From c768094bd1cbcdc97f6234766e2b27f070013bbd Mon Sep 17 00:00:00 2001 From: camillebrianceau <57992134+camillebrianceau@users.noreply.github.com> Date: Wed, 29 May 2024 18:51:02 +0200 Subject: [PATCH] Cb refactoring rebase (#597) * rebase refactoring on dev --- .github/workflows/test_cli.yml | 48 ++ .github/workflows/test_generate.yml | 53 +++ .github/workflows/test_interpret.yml | 53 +++ .github/workflows/test_predict.yml | 53 +++ .github/workflows/test_prepare_data.yml | 53 +++ .github/workflows/test_quality_checks.yml | 53 +++ .github/workflows/test_random_search.yml | 53 +++ .github/workflows/test_resume.yml | 53 +++ .github/workflows/test_train.yml | 53 +++ .github/workflows/test_transfer_learning.yml | 53 +++ .github/workflows/test_tsvtools.yml | 53 +++ .jenkins/Jenkinsfile | 465 ------------------- .jenkins/scripts/find_env.sh | 39 -- .jenkins/scripts/generate_wheels.sh | 31 -- poetry.lock | 51 +- 15 files changed, 605 insertions(+), 559 deletions(-) create mode 100644 .github/workflows/test_cli.yml create mode 100644 .github/workflows/test_generate.yml create mode 100644 .github/workflows/test_interpret.yml create mode 100644 .github/workflows/test_predict.yml create mode 100644 .github/workflows/test_prepare_data.yml create mode 100644 .github/workflows/test_quality_checks.yml create mode 100644 .github/workflows/test_random_search.yml create mode 100644 .github/workflows/test_resume.yml create mode 100644 .github/workflows/test_train.yml create mode 100644 .github/workflows/test_transfer_learning.yml create mode 100644 .github/workflows/test_tsvtools.yml delete mode 100644 .jenkins/Jenkinsfile delete mode 100755 .jenkins/scripts/find_env.sh delete mode 100755 .jenkins/scripts/generate_wheels.sh diff --git a/.github/workflows/test_cli.yml b/.github/workflows/test_cli.yml new file mode 100644 index 000000000..750f1cd00 --- /dev/null +++ b/.github/workflows/test_cli.yml @@ -0,0 +1,48 @@ +name: CLI Tests + +on: + push: + branches: ["dev", "refactoring"] + pull_request: + branches: ["dev", "refactoring"] + +permissions: + contents: read + +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' + cancel-in-progress: true + +env: + POETRY_VERSION: '1.8.3' + PYTHON_VERSION: '3.11' + +jobs: + test-cli: + if: github.event.pull_request.draft == false + runs-on: + - self-hosted + - Linux + - ubuntu + - cpu + steps: + - uses: actions/checkout@v4 + - uses: snok/install-poetry@v1 + with: + version: ${{ env.POETRY_VERSION }} + virtualenvs-create: false + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - name: Run CLI tests + run: | + make env.conda + source /builds/miniconda3/etc/profile.d/conda.sh + conda activate "${{ github.workspace }}"/env + make install + cd tests + poetry run pytest --verbose \ + --junitxml=./test-reports/test_cli_report.xml \ + --disable-warnings \ + --verbose \ + test_cli.py diff --git a/.github/workflows/test_generate.yml b/.github/workflows/test_generate.yml new file mode 100644 index 000000000..51ac863b2 --- /dev/null +++ b/.github/workflows/test_generate.yml @@ -0,0 +1,53 @@ +name: Generate Tests + +on: + push: + branches: ["dev", "refactoring"] + pull_request: + branches: ["dev", "refactoring"] + +permissions: + contents: read + +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' + cancel-in-progress: true + +env: + POETRY_VERSION: '1.8.3' + PYTHON_VERSION: '3.11' + +jobs: + test-generate: + if: github.event.pull_request.draft == false + runs-on: + - self-hosted + - Linux + - ubuntu + - cpu + steps: + - uses: actions/checkout@v4 + - uses: snok/install-poetry@v1 + with: + version: ${{ env.POETRY_VERSION }} + virtualenvs-create: false + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - name: Run tests for generate task + run: | + make env.conda + source /builds/miniconda3/etc/profile.d/conda.sh + conda activate "${{ github.workspace }}"/env + make install + cd tests + poetry run pytest --verbose \ + --junitxml=./test-reports/test_generate_report.xml \ + --disable-warnings \ + --verbose \ + --basetemp=$HOME/tmp/generate \ + --input_data_directory=/mnt/data/data_ci \ + test_generate.py + - name: Cleaning + run: | + rm -rf $HOME/tmp/generate diff --git a/.github/workflows/test_interpret.yml b/.github/workflows/test_interpret.yml new file mode 100644 index 000000000..0163bf583 --- /dev/null +++ b/.github/workflows/test_interpret.yml @@ -0,0 +1,53 @@ +name: Interpretation Tests (GPU) + +on: + push: + branches: ["dev", "refactoring"] + pull_request: + branches: ["dev", "refactoring"] + +permissions: + contents: read + +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' + cancel-in-progress: true + +env: + POETRY_VERSION: '1.8.3' + PYTHON_VERSION: '3.11' + +jobs: + test-interpret-gpu: + if: github.event.pull_request.draft == false + runs-on: + - self-hosted + - Linux + - ubuntu + - gpu + steps: + - uses: actions/checkout@v4 + - uses: snok/install-poetry@v1 + with: + version: ${{ env.POETRY_VERSION }} + virtualenvs-create: false + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - name: Run tests for Interpret task on GPU + run: | + make env.conda + source "${HOME}/miniconda3/etc/profile.d/conda.sh" + conda activate "${{ github.workspace }}"/env + make install + cd tests + poetry run pytest --verbose \ + --junitxml=./test-reports/test_interpret_report.xml \ + --disable-warnings \ + --verbose \ + --basetemp=$HOME/actions_runner_workdir/interpret \ + --input_data_directory=/mnt/data/clinicadl_data_ci/data_ci \ + test_interpret.py + - name: Cleaning + run: | + rm -rf $HOME/actions_runner_workdir/interpret/* diff --git a/.github/workflows/test_predict.yml b/.github/workflows/test_predict.yml new file mode 100644 index 000000000..8ec5976e4 --- /dev/null +++ b/.github/workflows/test_predict.yml @@ -0,0 +1,53 @@ +name: Predict Tests + +on: + push: + branches: ["dev", "refactoring"] + pull_request: + branches: ["dev", "refactoring"] + +permissions: + contents: read + +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' + cancel-in-progress: true + +env: + POETRY_VERSION: '1.8.3' + PYTHON_VERSION: '3.11' + +jobs: + test-predict: + if: github.event.pull_request.draft == false + runs-on: + - self-hosted + - Linux + - ubuntu + - cpu + steps: + - uses: actions/checkout@v4 + - uses: snok/install-poetry@v1 + with: + version: ${{ env.POETRY_VERSION }} + virtualenvs-create: false + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - name: Run tests for predict task + run: | + make env.conda + source /builds/miniconda3/etc/profile.d/conda.sh + conda activate "${{ github.workspace }}"/env + make install + cd tests + poetry run pytest --verbose \ + --junitxml=./test-reports/test_predict_report.xml \ + --disable-warnings \ + --verbose \ + --basetemp=$HOME/tmp/predict \ + --input_data_directory=/mnt/data/data_ci \ + test_predict.py + - name: Cleaning + run: | + rm -rf $HOME/tmp/predict/* diff --git a/.github/workflows/test_prepare_data.yml b/.github/workflows/test_prepare_data.yml new file mode 100644 index 000000000..8dccd217f --- /dev/null +++ b/.github/workflows/test_prepare_data.yml @@ -0,0 +1,53 @@ +name: Prepare data Tests + +on: + push: + branches: ["dev", "refactoring"] + pull_request: + branches: ["dev", "refactoring"] + +permissions: + contents: read + +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' + cancel-in-progress: true + +env: + POETRY_VERSION: '1.8.3' + PYTHON_VERSION: '3.11' + +jobs: + test-prepare-data: + if: github.event.pull_request.draft == false + runs-on: + - self-hosted + - Linux + - ubuntu + - cpu + steps: + - uses: actions/checkout@v4 + - uses: snok/install-poetry@v1 + with: + version: ${{ env.POETRY_VERSION }} + virtualenvs-create: false + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - name: Run tests for prepare data task + run: | + make env.conda + source /builds/miniconda3/etc/profile.d/conda.sh + conda activate "${{ github.workspace }}"/env + make install + cd tests + poetry run pytest --verbose \ + --junitxml=./test-reports/test_prepare_data_report.xml \ + --disable-warnings \ + --verbose \ + --basetemp=$HOME/tmp/prepare_data \ + --input_data_directory=/mnt/data/data_ci \ + test_prepare_data.py + - name: Cleaning + run: | + rm -rf $HOME/tmp/prepare_data/* diff --git a/.github/workflows/test_quality_checks.yml b/.github/workflows/test_quality_checks.yml new file mode 100644 index 000000000..1cf0414e2 --- /dev/null +++ b/.github/workflows/test_quality_checks.yml @@ -0,0 +1,53 @@ +name: Quality Check Tests + +on: + push: + branches: ["dev", "refactoring"] + pull_request: + branches: ["dev", "refactoring"] + +permissions: + contents: read + +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' + cancel-in-progress: true + +env: + POETRY_VERSION: '1.8.3' + PYTHON_VERSION: '3.11' + +jobs: + test-quality-check: + if: github.event.pull_request.draft == false + runs-on: + - self-hosted + - Linux + - ubuntu + - cpu + steps: + - uses: actions/checkout@v4 + - uses: snok/install-poetry@v1 + with: + version: ${{ env.POETRY_VERSION }} + virtualenvs-create: false + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - name: Run tests for Quality Check + run: | + make env.conda + source /builds/miniconda3/etc/profile.d/conda.sh + conda activate "${{ github.workspace }}"/env + make install + cd tests + poetry run pytest --verbose \ + --junitxml=./test-reports/test_quality_check_report.xml \ + --disable-warnings \ + --verbose \ + --basetemp=$HOME/tmp/quality_checks \ + --input_data_directory=/mnt/data/data_ci \ + test_qc.py + - name: Cleaning + run: | + rm -rf $HOME/tmp/quality_checks/* diff --git a/.github/workflows/test_random_search.yml b/.github/workflows/test_random_search.yml new file mode 100644 index 000000000..529f1fda1 --- /dev/null +++ b/.github/workflows/test_random_search.yml @@ -0,0 +1,53 @@ +name: Random Search Tests (GPU) + +on: + push: + branches: ["dev", "refactoring"] + pull_request: + branches: ["dev", "refactoring"] + +permissions: + contents: read + +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' + cancel-in-progress: true + +env: + POETRY_VERSION: '1.8.3' + PYTHON_VERSION: '3.11' + +jobs: + test-random-search-gpu: + if: github.event.pull_request.draft == false + runs-on: + - self-hosted + - Linux + - ubuntu + - gpu + steps: + - uses: actions/checkout@v4 + - uses: snok/install-poetry@v1 + with: + version: ${{ env.POETRY_VERSION }} + virtualenvs-create: false + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - name: Run Random Search tests on GPU + run: | + make env.conda + source "${HOME}/miniconda3/etc/profile.d/conda.sh" + conda activate "${{ github.workspace }}"/env + make install + cd tests + poetry run pytest --verbose \ + --junitxml=./test-reports/test_random_search_report.xml \ + --disable-warnings \ + --verbose \ + --basetemp=$HOME/actions_runner_workdir/random_search \ + --input_data_directory=/mnt/data/clinicadl_data_ci/data_ci \ + test_random_search.py + - name: Cleaning + run: | + rm -rf $HOME/actions_runner_workdir/random_search/* diff --git a/.github/workflows/test_resume.yml b/.github/workflows/test_resume.yml new file mode 100644 index 000000000..b789a21f6 --- /dev/null +++ b/.github/workflows/test_resume.yml @@ -0,0 +1,53 @@ +name: Resume Tests (GPU) + +on: + push: + branches: ["dev", "refactoring"] + pull_request: + branches: ["dev", "refactoring"] + +permissions: + contents: read + +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' + cancel-in-progress: true + +env: + POETRY_VERSION: '1.8.3' + PYTHON_VERSION: '3.11' + +jobs: + test-resume-gpu: + if: github.event.pull_request.draft == false + runs-on: + - self-hosted + - Linux + - ubuntu + - gpu + steps: + - uses: actions/checkout@v4 + - uses: snok/install-poetry@v1 + with: + version: ${{ env.POETRY_VERSION }} + virtualenvs-create: false + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - name: Run resume tests on GPU + run: | + make env.conda + source "${HOME}/miniconda3/etc/profile.d/conda.sh" + conda activate "${{ github.workspace }}"/env + make install + cd tests + poetry run pytest --verbose \ + --junitxml=./test-reports/test_resume_report.xml \ + --disable-warnings \ + --verbose \ + --basetemp=$HOME/actions_runner_workdir/resume \ + --input_data_directory=/mnt/data/clinicadl_data_ci/data_ci \ + test_resume.py + - name: Cleaning + run: | + rm -rf $HOME/actions_runner_workdir/resume/* diff --git a/.github/workflows/test_train.yml b/.github/workflows/test_train.yml new file mode 100644 index 000000000..a65a92a56 --- /dev/null +++ b/.github/workflows/test_train.yml @@ -0,0 +1,53 @@ +name: Train Tests (GPU) + +on: + push: + branches: ["dev", "refactoring"] + pull_request: + branches: ["dev", "refactoring"] + +permissions: + contents: read + +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' + cancel-in-progress: true + +env: + POETRY_VERSION: '1.8.3' + PYTHON_VERSION: '3.11' + +jobs: + test-train-gpu: + if: github.event.pull_request.draft == false + runs-on: + - self-hosted + - Linux + - ubuntu + - gpu + steps: + - uses: actions/checkout@v4 + - uses: snok/install-poetry@v1 + with: + version: ${{ env.POETRY_VERSION }} + virtualenvs-create: false + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - name: Run tests for Train on GPU + run: | + make env.conda + source "${HOME}/miniconda3/etc/profile.d/conda.sh" + conda activate "${{ github.workspace }}"/env + make install + cd tests + poetry run pytest --verbose \ + --junitxml=./test-reports/test_train_report.xml \ + --disable-warnings \ + --verbose \ + --basetemp=$HOME/actions_runner_workdir/train \ + --input_data_directory=/mnt/data/clinicadl_data_ci/data_ci \ + -k test_train + - name: Cleaning + run: | + rm -rf $HOME/actions_runner_workdir/train/* diff --git a/.github/workflows/test_transfer_learning.yml b/.github/workflows/test_transfer_learning.yml new file mode 100644 index 000000000..61238d4e1 --- /dev/null +++ b/.github/workflows/test_transfer_learning.yml @@ -0,0 +1,53 @@ +name: Transfer Learning Tests (GPU) + +on: + push: + branches: ["dev", "refactoring"] + pull_request: + branches: ["dev", "refactoring"] + +permissions: + contents: read + +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' + cancel-in-progress: true + +env: + POETRY_VERSION: '1.8.3' + PYTHON_VERSION: '3.11' + +jobs: + test-transfer-learning-gpu: + if: github.event.pull_request.draft == false + runs-on: + - self-hosted + - Linux + - ubuntu + - gpu + steps: + - uses: actions/checkout@v4 + - uses: snok/install-poetry@v1 + with: + version: ${{ env.POETRY_VERSION }} + virtualenvs-create: false + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - name: Run tests for Transfer Learning on GPU + run: | + make env.conda + source "${HOME}/miniconda3/etc/profile.d/conda.sh" + conda activate "${{ github.workspace }}"/env + make install + cd tests + poetry run pytest --verbose \ + --junitxml=./test-reports/test_transfer_learning_report.xml \ + --disable-warnings \ + --verbose \ + --basetemp=$HOME/actions_runner_workdir/transfer_learning \ + --input_data_directory=/mnt/data/clinicadl_data_ci/data_ci \ + test_transfer_learning.py + - name: Cleaning + run: | + rm -rf $HOME/actions_runner_workdir/transfer_learning/* diff --git a/.github/workflows/test_tsvtools.yml b/.github/workflows/test_tsvtools.yml new file mode 100644 index 000000000..811c6d4f4 --- /dev/null +++ b/.github/workflows/test_tsvtools.yml @@ -0,0 +1,53 @@ +name: TSV Tools Tests + +on: + push: + branches: ["dev", "refactoring"] + pull_request: + branches: ["dev", "refactoring"] + +permissions: + contents: read + +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' + cancel-in-progress: true + +env: + POETRY_VERSION: '1.8.3' + PYTHON_VERSION: '3.11' + +jobs: + test-tsvtools: + if: github.event.pull_request.draft == false + runs-on: + - self-hosted + - Linux + - ubuntu + - cpu + steps: + - uses: actions/checkout@v4 + - uses: snok/install-poetry@v1 + with: + version: ${{ env.POETRY_VERSION }} + virtualenvs-create: false + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - name: Run tests for TSV tools + run: | + make env.conda + source /builds/miniconda3/etc/profile.d/conda.sh + conda activate "${{ github.workspace }}"/env + make install + cd tests + poetry run pytest --verbose \ + --junitxml=./test-reports/test_tsvtools_report.xml \ + --disable-warnings \ + --verbose \ + --basetemp=$HOME/tmp/tsv_tools \ + --input_data_directory=/mnt/data/data_ci \ + test_tsvtools.py + - name: Cleaning + run: | + rm -rf $HOME/tmp/tsv_tools/* diff --git a/.jenkins/Jenkinsfile b/.jenkins/Jenkinsfile deleted file mode 100644 index f7bd3dafb..000000000 --- a/.jenkins/Jenkinsfile +++ /dev/null @@ -1,465 +0,0 @@ -#!/usr/bin/env groovy - -// Continuous Integration script for clinicadl -// Author: mauricio.diaz@inria.fr - -pipeline { - options { - timeout(time: 1, unit: 'HOURS') - disableConcurrentBuilds(abortPrevious: true) - } - agent none - stages { - stage('Functional tests') { - failFast false - parallel { - stage('No GPU') { - agent { - label 'cpu' - } - environment { - CONDA_HOME = "$HOME/miniconda" - CONDA_ENV = "$WORKSPACE/env" - PATH = "$HOME/.local/bin:$PATH" - TMP_DIR = "$HOME/tmp" - INPUT_DATA_DIR = '/mnt/data/clinicadl_data_ci/data_ci' - } - stages { - stage('Build Env') { - steps { - echo 'Installing clinicadl sources in Linux...' - echo "My branch name is ${BRANCH_NAME}" - sh "echo 'My branch name is ${BRANCH_NAME}'" - sh 'printenv' - sh "echo 'Agent name: ${NODE_NAME}'" - sh ''' - set +x - source "${CONDA_HOME}/etc/profile.d/conda.sh" - make env.conda - conda activate "${CONDA_ENV}" - conda info - echo "Install clinicadl using poetry..." - cd $WORKSPACE - make env - # Show clinicadl help message - echo "Display clinicadl help message" - clinicadl --help - conda deactivate - ''' - } - } - stage('CLI tests Linux') { - steps { - catchError(buildResult: 'FAILURE', stageResult: 'UNSTABLE') { - echo 'Testing pipeline instantiation...' - sh 'echo "Agent name: ${NODE_NAME}"' - sh ''' - set +x - echo $WORKSPACE - source "${CONDA_HOME}/etc/profile.d/conda.sh" - conda activate "${CONDA_ENV}" - conda list - cd $WORKSPACE/tests - poetry run pytest \ - --junitxml=./test-reports/test_cli_report.xml \ - --verbose \ - --disable-warnings \ - test_cli.py - conda deactivate - ''' - } - } - } - stage('tsvtools tests Linux') { - steps { - catchError(buildResult: 'FAILURE', stageResult: 'UNSTABLE') { - echo 'Testing tsvtool tasks...' - sh "echo 'Agent name: ${NODE_NAME}'" - sh ''' - source "${CONDA_HOME}/etc/profile.d/conda.sh" - conda activate "${CONDA_ENV}" - cd $WORKSPACE/tests - poetry run pytest \ - --junitxml=./test-reports/test_tsvtool_report.xml \ - --verbose \ - --disable-warnings \ - --basetemp=$TMP_DIR \ - --input_data_directory=$INPUT_DATA_DIR \ - test_tsvtools.py - conda deactivate - ''' - } - } - post { - always { - junit 'tests/test-reports/test_tsvtool_report.xml' - } - success { - sh 'rm -rf ${TMP_DIR}/*' - } - } - } - stage('Quality check tests Linux') { - steps { - catchError(buildResult: 'FAILURE', stageResult: 'UNSTABLE') { - echo 'Testing quality check tasks...' - sh "echo 'Agent name: ${NODE_NAME}'" - sh ''' - source "${CONDA_HOME}/etc/profile.d/conda.sh" - conda activate "${CONDA_ENV}" - cd $WORKSPACE/tests - poetry run pytest \ - --junitxml=./test-reports/test_quality_check_report.xml \ - --verbose \ - --disable-warnings \ - --basetemp=$TMP_DIR \ - --input_data_directory=$INPUT_DATA_DIR \ - test_qc.py - conda deactivate - ''' - } - } - post { - always { - junit 'tests/test-reports/test_quality_check_report.xml' - } - success { - sh 'rm -rf ${TMP_DIR}/*' - } - } - } - stage('Generate tests Linux') { - steps { - catchError(buildResult: 'FAILURE', stageResult: 'UNSTABLE') { - echo 'Testing generate task...' - sh "echo 'Agent name: ${NODE_NAME}'" - sh ''' - source "${CONDA_HOME}/etc/profile.d/conda.sh" - conda activate "${CONDA_ENV}" - cd $WORKSPACE/tests - poetry run pytest \ - --junitxml=./test-reports/test_generate_report.xml \ - --verbose \ - --disable-warnings \ - --basetemp=$TMP_DIR \ - --input_data_directory=$INPUT_DATA_DIR \ - test_generate.py - conda deactivate - ''' - } - } - post { - always { - junit 'tests/test-reports/test_generate_report.xml' - } - success { - sh 'rm -rf ${TMP_DIR}/*' - } - } - } - stage('Prepare data tests Linux') { - steps { - catchError(buildResult: 'FAILURE', stageResult: 'UNSTABLE') { - echo 'Testing prepare_data task...' - sh "echo 'Agent name: ${NODE_NAME}'" - sh ''' - source "${CONDA_HOME}/etc/profile.d/conda.sh" - conda activate "${CONDA_ENV}" - cd $WORKSPACE/tests - poetry run pytest \ - --junitxml=./test-reports/test_prepare_data_report.xml \ - --verbose \ - --disable-warnings \ - --basetemp=$TMP_DIR \ - --input_data_directory=$INPUT_DATA_DIR \ - test_prepare_data.py - conda deactivate - ''' - } - } - post { - always { - junit 'tests/test-reports/test_prepare_data_report.xml' - } - success { - sh 'rm -rf ${TMP_DIR}/*' - } - } - } - stage('Predict tests Linux') { - steps { - catchError(buildResult: 'FAILURE', stageResult: 'UNSTABLE') { - echo 'Testing predict...' - sh "echo 'Agent name: ${NODE_NAME}'" - sh ''' - source "${CONDA_HOME}/etc/profile.d/conda.sh" - conda activate "${CONDA_ENV}" - cd $WORKSPACE/tests - poetry run pytest \ - --junitxml=./test-reports/test_predict_report.xml \ - --verbose \ - --disable-warnings \ - --basetemp=$TMP_DIR \ - --input_data_directory=$INPUT_DATA_DIR \ - test_predict.py - conda deactivate - ''' - } - } - post { - always { - junit 'tests/test-reports/test_predict_report.xml' - } - success { - sh 'rm -rf ${TMP_DIR}/*' - } - } - } - // stage('Meta-maps analysis') { - // environment { - // PATH = "$HOME/miniconda3/bin:$HOME/miniconda/bin:$PATH" - // } - // steps { - // echo 'Testing maps-analysis task...' - // sh 'echo "Agent name: ${NODE_NAME}"' - // sh '''#!/usr/bin/env bash - // set +x - // eval "$(conda shell.bash hook)" - // conda activate "${WORKSPACE}/env" - // cd $WORKSPACE/tests - // pytest \ - // --junitxml=./test-reports/test_meta-analysis_report.xml \ - // --verbose \ - // --disable-warnings \ - // test_meta_maps.py - // conda deactivate - // ''' - // } - // post { - // always { - // junit 'tests/test-reports/test_meta-analysis_report.xml' - // sh 'rm -rf $WORKSPACE/tests/data/dataset' - // } - // } - // } - } - post { - // Clean after build - cleanup { - cleanWs(deleteDirs: true, - notFailBuild: true, - patterns: [[pattern: 'env', type: 'INCLUDE']]) - } - } - } - stage('GPU') { - agent { - label 'gpu' - } - environment { - CONDA_HOME = "$HOME/miniconda3" - CONDA_ENV = "$WORKSPACE/env" - PATH = "$HOME/.local/bin:$PATH" - TMP_DIR = "$HOME/tmp" - INPUT_DATA_DIR = '/mnt/data/clinicadl_data_ci/data_ci' - } - stages { - stage('Build Env') { - steps { - echo 'Installing clinicadl sources in Linux...' - echo "My branch name is ${BRANCH_NAME}" - sh "echo 'My branch name is ${BRANCH_NAME}'" - sh 'printenv' - sh "echo 'Agent name: ${NODE_NAME}'" - sh '''#!/usr/bin/env bash - source "${CONDA_HOME}/etc/profile.d/conda.sh" - make env.conda - conda activate "${CONDA_ENV}" - conda info - echo "Install clinicadl using poetry..." - cd $WORKSPACE - make env - # Show clinicadl help message - echo "Display clinicadl help message" - clinicadl --help - conda deactivate - ''' - } - } - stage('Train tests Linux') { - steps { - catchError(buildResult: 'FAILURE', stageResult: 'UNSTABLE') { - echo 'Testing train task...' - sh "echo 'Agent name: ${NODE_NAME}'" - sh '''#!/usr/bin/env bash - source "${CONDA_HOME}/etc/profile.d/conda.sh" - conda activate "${CONDA_ENV}" - clinicadl --help - cd $WORKSPACE/tests - poetry run pytest \ - --junitxml=./test-reports/test_train_report.xml \ - --verbose \ - --disable-warnings \ - --basetemp=$TMP_DIR \ - --input_data_directory=$INPUT_DATA_DIR \ - -k "test_train" - conda deactivate - ''' - } - } - post { - always { - junit 'tests/test-reports/test_train_report.xml' - } - success { - sh 'rm -rf ${TMP_DIR}/*' - } - } - } - stage('Transfer learning tests Linux') { - steps { - catchError(buildResult: 'FAILURE', stageResult: 'UNSTABLE') { - echo 'Testing transfer learning...' - sh "echo 'Agent name: ${NODE_NAME}'" - sh '''#!/usr/bin/env bash - source "${CONDA_HOME}/etc/profile.d/conda.sh" - conda activate "${CONDA_ENV}" - clinicadl --help - cd $WORKSPACE/tests - poetry run pytest \ - --junitxml=./test-reports/test_transfer_learning_report.xml \ - --verbose \ - --disable-warnings \ - --basetemp=$TMP_DIR \ - --input_data_directory=$INPUT_DATA_DIR \ - test_transfer_learning.py - conda deactivate - ''' - } - } - post { - always { - junit 'tests/test-reports/test_transfer_learning_report.xml' - } - success { - sh 'rm -rf ${TMP_DIR}/*' - } - } - } - stage('Resume tests Linux') { - steps { - catchError(buildResult: 'FAILURE', stageResult: 'UNSTABLE') { - echo 'Testing resume...' - sh "echo 'Agent name: ${NODE_NAME}'" - sh '''#!/usr/bin/env bash - source "${CONDA_HOME}/etc/profile.d/conda.sh" - conda activate "${CONDA_ENV}" - clinicadl --help - cd $WORKSPACE/tests - poetry run pytest \ - --junitxml=./test-reports/test_resume_report.xml \ - --verbose \ - --disable-warnings \ - --basetemp=$TMP_DIR \ - --input_data_directory=$INPUT_DATA_DIR \ - test_resume.py - conda deactivate - ''' - } - } - post { - always { - junit 'tests/test-reports/test_resume_report.xml' - } - success { - sh 'rm -rf ${TMP_DIR}/*' - } - } - } - stage('Interpretation tests Linux') { - steps { - catchError(buildResult: 'FAILURE', stageResult: 'UNSTABLE') { - echo 'Testing interpret task...' - sh "echo 'Agent name: ${NODE_NAME}'" - sh '''#!/usr/bin/env bash - set +x - source "${CONDA_HOME}/etc/profile.d/conda.sh" - conda activate "${CONDA_ENV}" - clinicadl --help - cd $WORKSPACE/tests - poetry run pytest \ - --junitxml=./test-reports/test_interpret_report.xml \ - --verbose \ - --disable-warnings \ - --basetemp=$TMP_DIR \ - --input_data_directory=$INPUT_DATA_DIR \ - test_interpret.py - conda deactivate - ''' - } - } - post { - always { - junit 'tests/test-reports/test_interpret_report.xml' - } - success { - sh 'rm -rf ${TMP_DIR}/*' - } - } - } - stage('Random search tests Linux') { - steps { - catchError(buildResult: 'FAILURE', stageResult: 'UNSTABLE') { - echo 'Testing random search...' - sh "echo 'Agent name: ${NODE_NAME}'" - sh '''#!/usr/bin/env bash - set +x - source "${CONDA_HOME}/etc/profile.d/conda.sh" - conda activate "${CONDA_ENV}" - clinicadl --help - cd $WORKSPACE/tests - poetry run pytest \ - --junitxml=./test-reports/test_random_search_report.xml \ - --verbose \ - --disable-warnings \ - --basetemp=$TMP_DIR \ - --input_data_directory=$INPUT_DATA_DIR \ - test_random_search.py - conda deactivate - ''' - } - } - post { - always { - junit 'tests/test-reports/test_random_search_report.xml' - } - success { - sh 'rm -rf ${TMP_DIR}/*' - } - } - } - } - post { - // Clean after build - cleanup { - cleanWs(deleteDirs: true, - notFailBuild: true, - patterns: [[pattern: 'env', type: 'INCLUDE']]) - } - } - } - } - } - } -// post { -// failure { -// mail to: 'clinicadl-ci@inria.fr', -// subject: "Failed Pipeline: ${currentBuild.fullDisplayName}", -// body: "Something is wrong with ${env.BUILD_URL}" -// mattermostSend( -// color: "#FF0000", -// message: "ClinicaDL Build FAILED: ${env.JOB_NAME} #${env.BUILD_NUMBER} (<${env.BUILD_URL}|Link to build>)" -// ) -// } -// } -} diff --git a/.jenkins/scripts/find_env.sh b/.jenkins/scripts/find_env.sh deleted file mode 100755 index a68fff821..000000000 --- a/.jenkins/scripts/find_env.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -# A shell script to launch clinica in CI machines - -# Name of the Conda environment according to the branch -CLINICA_ENV_BRANCH="clinicadl_test" - -set -e -set +x - -ENV_EXISTS=0 -# Verify that the conda environment corresponding to the branch exists, otherwise -# create it. -ENVS=$(conda env list | awk '{print $1}' ) -echo $ENVS - -for ENV in $ENVS -do - if [[ "$ENV " == *"$CLINICA_ENV_BRANCH "* ]] - then - echo "Find Conda environment named $ENV, continue." - conda activate $CLINICA_ENV_BRANCH - cd $WORKSPACE/ - poetry install - conda deactivate - ENV_EXISTS=1 - break - fi; -done -if [ "$ENV_EXISTS" = 0 ]; then - echo "Conda env $CLINICA_ENV_BRANCH not found... Creating" - conda create -y -f environment.yml - echo "Conda env $CLINICA_ENV_BRANCH was created." - conda activate $CLINICA_ENV_BRANCH - cd $WORKSPACE/ - poetry install - echo "ClinicaDL has been installed in $CLINICA_ENV_BRANCH." - conda deactivate - cd $WORKSPACE -fi diff --git a/.jenkins/scripts/generate_wheels.sh b/.jenkins/scripts/generate_wheels.sh deleted file mode 100755 index 326d55074..000000000 --- a/.jenkins/scripts/generate_wheels.sh +++ /dev/null @@ -1,31 +0,0 @@ -#! /bin/sh - -#--------------------------------------# -# ClinicaDL package creations ( wheel) -#--------------------------------------# -# -# WARNING: Activate a conda environment with the right pip version. -# Use at your own risk. - - -CURRENT_DIR=$(pwd) -echo $CURRENT_DIR - -# ensure we are in the right dir -SCRIPT_DIR=`(dirname $0)` -cd "$SCRIPT_DIR" -echo "Entering ${SCRIPT_DIR}/../../" -cd "${SCRIPT_DIR}/../../" -ls - -# clean pycache stuff -rm -rf dist build clinicadl.egg-info/ -find -name "*__pycache__*" -exec rm {} \-rf \; -find -name "*.pyc*" -exec rm {} \-rf \; - -set -o errexit -set -e -# generate wheel -poetry build -# come back to directory of -cd $CURRENT_DIR diff --git a/poetry.lock b/poetry.lock index 19da6f244..c419d290e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -999,22 +999,23 @@ protobuf = ["grpcio-tools (>=1.62.1)"] [[package]] name = "gunicorn" -version = "21.2.0" +version = "22.0.0" description = "WSGI HTTP Server for UNIX" optional = false -python-versions = ">=3.5" +python-versions = ">=3.7" files = [ - {file = "gunicorn-21.2.0-py3-none-any.whl", hash = "sha256:3213aa5e8c24949e792bcacfc176fef362e7aac80b76c56f6b5122bf350722f0"}, - {file = "gunicorn-21.2.0.tar.gz", hash = "sha256:88ec8bff1d634f98e61b9f65bc4bf3cd918a90806c6f5c48bc5603849ec81033"}, + {file = "gunicorn-22.0.0-py3-none-any.whl", hash = "sha256:350679f91b24062c86e386e198a15438d53a7a8207235a78ba1b53df4c4378d9"}, + {file = "gunicorn-22.0.0.tar.gz", hash = "sha256:4a0b436239ff76fb33f11c07a16482c521a7e09c1ce3cc293c2330afe01bec63"}, ] [package.dependencies] packaging = "*" [package.extras] -eventlet = ["eventlet (>=0.24.1)"] +eventlet = ["eventlet (>=0.24.1,!=0.36.0)"] gevent = ["gevent (>=1.4.0)"] setproctitle = ["setproctitle"] +testing = ["coverage", "eventlet", "gevent", "pytest", "pytest-cov"] tornado = ["tornado (>=0.2)"] [[package]] @@ -1149,13 +1150,13 @@ files = [ [[package]] name = "jinja2" -version = "3.1.3" +version = "3.1.4" description = "A very fast and expressive template engine." optional = false python-versions = ">=3.7" files = [ - {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, - {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, + {file = "jinja2-3.1.4-py3-none-any.whl", hash = "sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d"}, + {file = "jinja2-3.1.4.tar.gz", hash = "sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369"}, ] [package.dependencies] @@ -1771,13 +1772,13 @@ files = [ [[package]] name = "mlflow" -version = "2.11.3" -description = "MLflow: A Platform for ML Development and Productionization" +version = "2.12.2" +description = "MLflow is an open source platform for the complete machine learning lifecycle" optional = false python-versions = ">=3.8" files = [ - {file = "mlflow-2.11.3-py3-none-any.whl", hash = "sha256:6af105162b6f7f2a69cace48c2af7adbecfada5e6386f1ce7d1bbbffd6e09953"}, - {file = "mlflow-2.11.3.tar.gz", hash = "sha256:621b7e311e890b79719c2e7777286d8e08d38dd04d5ab080966990bd4a55febb"}, + {file = "mlflow-2.12.2-py3-none-any.whl", hash = "sha256:38dd04710fe64ee8229b7233b4d91db32c3ff887934c40d926246a566c886c0b"}, + {file = "mlflow-2.12.2.tar.gz", hash = "sha256:d712f1af9d44f1eb9e1baee8ca64f7311e185b7572fc3c1e0a83a4c8ceff6aad"}, ] [package.dependencies] @@ -1789,7 +1790,7 @@ entrypoints = "<1" Flask = "<4" gitpython = ">=3.1.9,<4" graphene = "<4" -gunicorn = {version = "<22", markers = "platform_system != \"Windows\""} +gunicorn = {version = "<23", markers = "platform_system != \"Windows\""} importlib-metadata = ">=3.7.0,<4.7.0 || >4.7.0,<8" Jinja2 = [ {version = ">=2.11,<4", markers = "platform_system != \"Windows\""}, @@ -1798,7 +1799,7 @@ Jinja2 = [ markdown = ">=3.3,<4" matplotlib = "<4" numpy = "<2" -packaging = "<24" +packaging = "<25" pandas = "<3" protobuf = ">=3.12.0,<5" pyarrow = ">=4.0.0,<16" @@ -3114,13 +3115,13 @@ files = [ [[package]] name = "requests" -version = "2.31.0" +version = "2.32.0" description = "Python HTTP for Humans." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, - {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, + {file = "requests-2.32.0-py3-none-any.whl", hash = "sha256:f2c3881dddb70d056c5bd7600a4fae312b2a300e39be6a118d30b90bd27262b5"}, + {file = "requests-2.32.0.tar.gz", hash = "sha256:fa5490319474c82ef1d2c9bc459d3652e3ae4ef4c4ebdd18a21145a47ca4b6b8"}, ] [package.dependencies] @@ -3882,13 +3883,14 @@ scipy = ["scipy"] [[package]] name = "tqdm" -version = "4.66.2" + +version = "4.66.3" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.7" files = [ - {file = "tqdm-4.66.2-py3-none-any.whl", hash = "sha256:1ee4f8a893eb9bef51c6e35730cebf234d5d0b6bd112b0271e10ed7c24a02bd9"}, - {file = "tqdm-4.66.2.tar.gz", hash = "sha256:6cd52cdf0fef0e0f543299cfc96fec90d7b8a7e88745f411ec33eb44d5ed3531"}, + {file = "tqdm-4.66.3-py3-none-any.whl", hash = "sha256:4f41d54107ff9a223dca80b53efe4fb654c67efaba7f47bada3ee9d50e05bd53"}, + {file = "tqdm-4.66.3.tar.gz", hash = "sha256:23097a41eba115ba99ecae40d06444c15d1c0c698d527a01c6c8bd1c5d0647e5"}, ] [package.dependencies] @@ -4099,13 +4101,14 @@ watchmedo = ["PyYAML (>=3.10)"] [[package]] name = "werkzeug" -version = "3.0.2" +version = "3.0.3" description = "The comprehensive WSGI web application library." optional = false python-versions = ">=3.8" files = [ - {file = "werkzeug-3.0.2-py3-none-any.whl", hash = "sha256:3aac3f5da756f93030740bc235d3e09449efcf65f2f55e3602e1d851b8f48795"}, - {file = "werkzeug-3.0.2.tar.gz", hash = "sha256:e39b645a6ac92822588e7b39a692e7828724ceae0b0d702ef96701f90e70128d"}, + + {file = "werkzeug-3.0.3-py3-none-any.whl", hash = "sha256:fc9645dc43e03e4d630d23143a04a7f947a9a3b5727cd535fdfe155a17cc48c8"}, + {file = "werkzeug-3.0.3.tar.gz", hash = "sha256:097e5bfda9f0aba8da6b8545146def481d06aa7d3266e7448e2cccf67dd8bd18"}, ] [package.dependencies]