diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml
index 752ebc3f..9052a6f3 100644
--- a/.github/workflows/docs.yaml
+++ b/.github/workflows/docs.yaml
@@ -1,8 +1,7 @@
---
-name: Publish Develop Docs
+name: Docs
on:
- pull_request:
push:
branches:
- main
@@ -30,6 +29,8 @@ jobs:
run: |
git config user.email "action@github.com"
git config user.name "GitHub Action"
- - name: Build Documentation
+ - name: Deploy Documentation
run: |
- nox -s docs -- gh-deploy --force --remote-branch gh-pages
+ git fetch origin gh-pages:gh-pages
+ nox -s docs -- deploy --update-aliases dev
+ git push origin gh-pages
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index b3ffb602..5c652f87 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -9,6 +9,9 @@ on:
jobs:
release-pypi:
runs-on: ubuntu-latest
+ permissions:
+ id-token: write
+ contents: write
steps:
- uses: actions/checkout@v4
- name: Setting up PDM
@@ -41,20 +44,11 @@ jobs:
git config --local user.name "GitHub Action"
git fetch origin gh-pages:gh-pages
tag="${{ github.ref_name }}"
- DOC_VERSION="0.0.0dev5"
+ DOC_VERSION=${tag%.*}
nox -s deploy_docs -- --alias-type=copy --update-aliases "$DOC_VERSION" latest
git push origin gh-pages
- # - name: Publish package distributions to PyPI
- # run: pdm publish --no-build
- # env:
- # PDM_PUBLISH_USERNAME: ${{ secrets.PYPI_USERNAME }}
- # PDM_PUBLISH_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
- # - name: Create Release
- # uses: actions/create-release@main
- # env:
- # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- # with:
- # tag_name: ${{ github.ref }}
- # release_name: v${{ github.ref }}
- # draft: true
- # prerelease: ${{ steps.check_version.outputs.PRERELEASE }}
+ - name: Publish package distributions to PyPI
+ run: nox -s publish -- --no-build
+ env:
+ PDM_PUBLISH_USERNAME: ${{ secrets.PYPI_USERNAME }}
+ PDM_PUBLISH_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
diff --git a/README.md b/README.md
index 73d2dea9..c7a067b3 100644
--- a/README.md
+++ b/README.md
@@ -1,77 +1,119 @@
----
-
-### _About_
+
-`eva` is [kaiko.ai](https://kaiko.ai/)'s evaluation framework for oncology foundation models (FMs). Check out the documentation (LINK TO BE ADDED) for more information.
+_`eva`_ is an evaluation framework for oncology foundation models (FMs) by [kaiko.ai](https://kaiko.ai/). Check out the [documentation](https://kaiko-ai.github.io/eva/) for more information.
+### Highlights:
+- Easy and reliable benchmark of Oncology FMs
+- Automatic embedding inference and evaluation of a downstream task
+- Native support of popular medical [datasets](https://kaiko-ai.github.io/eva/dev/datasets/) and models
+- Produce statistics over multiple evaluation fits and multiple metrics
## Installation
-*Note: this section will be revised for the public package when publishing eva*
+Simple installation from PyPI:
+```sh
+# to install the core version only
+pip install kaiko-eva
-- Create and activate a virtual environment with Python 3.10+
+# to install the expanded `vision` version
+pip install 'kaiko-eva[vision]'
-- Install *eva* and the *eva-vision* package with:
+# to install everything
+pip install 'kaiko-eva[all]'
+```
+To install the latest version of the `main` branch:
+```sh
+pip install "kaiko-eva[all] @ git+https://github.com/kaiko-ai/eva.git"
```
-pip install 'kaiko-eva[vision]'
+
+You can verify that the installation was successful by executing:
+```sh
+eva --version
```
-- To be able to use the existing configs, download them from the [*eva* GitHub repo](https://github.com/kaiko-ai/eva/tree/main) and move them to directory where you installed *eva*.
+## How To Use
-### Run *eva*
+_eva_ can be used directly from the terminal as a CLI tool as follows:
+```sh
+eva {fit,predict,predict_fit} --config url/or/path/to/the/config.yaml
+```
-Now you can run a complete *eva* workflow, for example with:
+For example, to perform a downstream evaluation of DINO ViT-S/16 on the BACH dataset with linear probing by first inferring the embeddings and performing 5 sequential fits, execute:
+```sh
+eva predict_fit --config https://raw.githubusercontent.com/kaiko-ai/eva/main/configs/vision/dino_vit/offline/bach.yaml
```
-eva fit --config configs/vision/dino_vit/online/bach.yaml
+
+> [!NOTE]
+> All the datasets that support automatic download in the repo have by default the option to automatically download set to false. For automatic download you have to manually set download=true.
+
+
+To view all the possibles, execute:
+```sh
+eva --help
```
-This will:
- - Download and extract the dataset, if it has not been downloaded before.
- - Fit a model consisting of the frozen FM-backbone and a classification head on the train split.
- - Evaluate the trained model on the validation split and report the results.
+For more information, please refer to the [documentation](https://kaiko-ai.github.io/eva/dev/user-guide/tutorials/offline_vs_online/) and [tutorials](https://kaiko-ai.github.io/eva/dev/user-guide/advanced/replicate_evaluations/).
-For more information, documentation and tutorials, refer to the documentation (LINK TO BE ADDED).
+## Benchmarks
-## Datasets
+In this section you will find model benchmarks which were generated with _eva_.
-The following datasets are supported natively:
+### Table I: WSI patch-level benchmark
-### Vision
+
+
+
+
+| Model | BACH | CRC | MHIST | PCam/val | PCam/test |
+|--------------------------------------------------|-------|-------|-------|----------|-----------|
+| ViT-S/16 _(random)_ [1] | 0.410 | 0.617 | 0.501 | 0.753 | 0.728 |
+| ViT-S/16 _(ImageNet)_ [1] | 0.695 | 0.935 | 0.831 | 0.864 | 0.849 |
+| ViT-B/8 _(ImageNet)_ [1] | 0.710 | 0.939 | 0.814 | 0.870 | 0.856 |
+| DINO(p=16) [2] | 0.801 | 0.934 | 0.768 | 0.889 | 0.895 |
+| Phikon [3] | 0.725 | 0.935 | 0.777 | 0.912 | 0.915 |
+| ViT-S/16 _(kaiko.ai)_ [4] | 0.797 | 0.943 | 0.828 | 0.903 | 0.893 |
+| ViT-S/8 _(kaiko.ai)_ [4] | 0.834 | 0.946 | 0.832 | 0.897 | 0.887 |
+| ViT-B/16 _(kaiko.ai)_ [4] | 0.810 | 0.960 | 0.826 | 0.900 | 0.898 |
+| ViT-B/8 _(kaiko.ai)_ [4] | 0.865 | 0.956 | 0.809 | 0.913 | 0.921 |
+| ViT-L/14 _(kaiko.ai)_ [4] | 0.870 | 0.930 | 0.809 | 0.908 | 0.898 |
+
+_Table I: Linear probing evaluation of FMs on patch-level downstream datasets.
We report averaged balanced accuracy
+over 5 runs, with an average standard deviation of ±0.003._
-#### Patch-level pathology datasets:
- - [BACH](./docs/datasets/bach.md)
- - [CRC](./docs/datasets/crc.md)
- - [MHIST](./docs/datasets/mhist.md)
- - [PatchCamelyon](./docs/datasets/patch_camelyon.md)
+
+
+
-#### Radiology datasets:
- - [TotalSegmentator](./docs/datasets/total_segmentator.md)
+_References_:
+1. _"Emerging properties in self-supervised vision transformers”_
+2. _"Benchmarking self-supervised learning on diverse pathology datasets”_
+3. _"Scaling self-supervised learning for histopathology with masked image modeling”_
+4. _"Towards Training Large-Scale Pathology Foundation Models: from TCGA to Hospital Scale”_
## Contributing
@@ -79,7 +121,27 @@ _eva_ is an open source project and welcomes contributions of all kinds. Please
All contributors must follow the [code of conduct](./docs/CODE_OF_CONDUCT.md).
+
+## Acknowledgements
+
+Our codebase is built using multiple opensource contributions
+
+
+
+[](https://github.com/pre-commit/pre-commit)
+[](https://pytorch.org/get-started/locally/)
+[](https://pytorchlightning.ai/)
+[](https://black.readthedocs.io/en/stable/)
+[](https://pycqa.github.io/isort/)
+[](https://github.com/astral-sh/ruff)
+[](https://microsoft.github.io/pyright/)
+[](https://pdm-project.org)
+[](https://github.com/wntrblm/nox)
+[](https://squidfunk.github.io/mkdocs-material/)
+
+
+
---
-
+
-
-
- 
+
+ 
+
+
+ 
-
- 
+
+ 
-
-
- User Guide •
- Datasets •
- Reference API
-
+
+#
+
_Oncology FM Evaluation Framework by [kaiko.ai](https://www.kaiko.ai/)_
-With the first release, ***eva*** supports performance evaluation for vision Foundation Models ("FMs") and supervised machine learning models on WSI-patch-level image classification- and radiology (CT-scans) segmentation tasks.
+With the first release, ***eva*** supports performance evaluation for vision Foundation Models ("FMs") and supervised machine learning models on WSI-patch-level image classification task. Support for radiology (CT-scans) segmentation tasks will be added soon.
-The goal of this project is to provide the open-source community with an easy-to-use framework that follows industry best practices to deliver a robust, reproducible and fair evaluation benchmark across FMs of different sizes and architectures.
+With *eva* we provide the open-source community with an easy-to-use framework that follows industry best practices to deliver a robust, reproducible and fair evaluation benchmark across FMs of different sizes and architectures.
Support for additional modalities and tasks will be added in future releases.
@@ -52,39 +51,42 @@ Supported datasets & tasks include:
*Radiology datasets*
-- **[TotalSegmentator](datasets/total_segmentator.md)**: radiology/CT-scan for segmentation of anatomical structures
-
-More datasets & downstream task types will be added in future releases.
+- **[TotalSegmentator](datasets/total_segmentator.md)**: radiology/CT-scan for segmentation of anatomical structures (*support coming soon*)
To evaluate FMs, *eva* provides support for different model-formats, including models trained with PyTorch, models available on HuggingFace and ONNX-models. For other formats custom wrappers can be implemented.
### 2. Evaluate ML models on your own dataset & task
-If you have your own labelled dataset, all that is needed is to implement a dataset class tailored to your source data. Start from one of our out-of-the box provided dataset classes, adapt it to your data and run *eva* to see how different FMs perform on your task.
+If you have your own labeled dataset, all that is needed is to implement a dataset class tailored to your source data. Start from one of our out-of-the box provided dataset classes, adapt it to your data and run *eva* to see how different FMs perform on your task.
## Evaluation results
-We evaluated the following FMs on the 4 supported WSI-patch-level image classification tasks:
+We evaluated the following FMs on the 4 supported WSI-patch-level image classification tasks. On the table below we report *Balanced Accuracy* for binary & multiclass tasks and show the average performance & standard deviation over 5 runs.
+
+
+
-| FM-backbone | pretraining | PCam - val* | PCam - test* | BACH - val* | CRC - val* | MHIST - val* |
-|-----------------------------|-------------|------------------|-----------------|-----------------|-----------------|--------------|
-| DINO ViT-S16 | N/A | 0.765 (±0.004) | 0.726 (±0.003) | 0.416 (±0.014) | 0.643 (±0.005) | 0.551 (±0.017)|
-| DINO ViT-S16 | ImageNet | 0.871 (±0.004) | 0.856 (±0.005) | 0.673 (±0.005) | 0.936 (±0.001) | 0.823 (±0.006)|
-| DINO ViT-B8 | ImageNet | 0.872 (±0.004) | 0.854 (±0.002) | 0.704 (±0.008) | 0.942 (±0.001) | 0.813 (±0.003)|
-| Lunit - ViT-S16 | TCGA | 0.89 (±0.001) | 0.897 (±0.003) | 0.765 (±0.011) | 0.936 (±0.001)| 0.762 (±0.004)|
-| Owkin - iBOT ViT-B16 | TCGA | **0.914 (±0.002)** | **0.919 (±0.009)** | 0.717 (±0.004) | 0.938 (±0.001)| 0.799 (±0.003)|
-| kaiko.ai - DINO ViT-S16 | TCGA | 0.911 (±0.002) | 0.899 (±0.002) | 0.773 (±0.007) | **0.954 (±0.002)** | **0.829 (±0.004)**|
-| kaiko.ai - DINO ViT-B8 | TCGA | 0.902 (±0.002) | 0.887 (±0.004) | **0.798 (±0.007)** | 0.950 (±0.003) | 0.803 (±0.004)|
-| kaiko.ai - DINOv2 ViT-L14 | TCGA | 0.900 (±0.002) | 0.896 (±0.001) | 0.768 (±0.006) | 0.945 (±0.001) | 0.777 (±0.008)|
+| FM-backbone | pretraining | BACH | CRC | MHIST | PCam/val | PCam/test |
+|-----------------------------|-------------|------------------ |----------------- |----------------- |----------------- |-------------- |
+| DINO ViT-S16 | N/A | 0.410 (±0.009) | 0.617 (±0.008) | 0.501 (±0.004) | 0.753 (±0.002) | 0.728 (±0.003) |
+| DINO ViT-S16 | ImageNet | 0.695 (±0.004) | 0.935 (±0.003) | 0.831 (±0.002) | 0.864 (±0.007) | 0.849 (±0.007) |
+| DINO ViT-B8 | ImageNet | 0.710 (±0.007) | 0.939 (±0.001) | 0.814 (±0.003) | 0.870 (±0.003) | 0.856 (±0.004) |
+| Lunit - ViT-S16 | TCGA | 0.801 (±0.005) | 0.934 (±0.001) | 0.768 (±0.004) | 0.889 (±0.002) | 0.895 (±0.006) |
+| Owkin - iBOT ViT-B16 | TCGA | 0.725 (±0.004) | 0.935 (±0.001) | 0.777 (±0.005) | 0.912 (±0.002) | 0.915 (±0.003) |
+| kaiko.ai - DINO ViT-S16 | TCGA | 0.797 (±0.003) | 0.943 (±0.001) | 0.828 (±0.003) | 0.903 (±0.001) | 0.893 (±0.005) |
+| kaiko.ai - DINO ViT-S8 | TCGA | 0.834 (±0.012) | 0.946 (±0.002) | **0.832 (±0.006)** | 0.897 (±0.001) | 0.887 (±0.002) |
+| kaiko.ai - DINO ViT-B16 | TCGA | 0.810 (±0.008) | **0.960 (±0.001)** | 0.826 (±0.003) | 0.900 (±0.002) | 0.898 (±0.003) |
+| kaiko.ai - DINO ViT-B8 | TCGA | 0.865 (±0.019) | 0.956 (±0.001) | 0.809 (±0.021) | **0.913 (±0.001)** | **0.921 (±0.002)**|
+| kaiko.ai - DINOv2 ViT-L14 | TCGA | **0.870 (±0.005)**| 0.930 (±0.001) | 0.809 (±0.001) | 0.908 (±0.001) | 0.898 (±0.002) |
-\* Metric in table: *Balanced Accuracy* (for binary & multiclass). The table shows the average performance & standard deviation over 5 runs.
+
The runs use the default setup described in the section below.
*eva* trains the decoder on the "train" split and uses the "validation" split for monitoring, early stopping and checkpoint selection. Evaluation results are reported on the "validation" split and, if available, on the "test" split.
-For more details on the FM-backbones and instructions to replicate the results, please refer to the [Replicate evaluations](user-guide/advanced/replicate_evaluations.md).
+For more details on the FM-backbones and instructions to replicate the results, check out [Replicate evaluations](user-guide/advanced/replicate_evaluations.md).
## Evaluation setup
diff --git a/docs/user-guide/advanced/replicate_evaluations.md b/docs/user-guide/advanced/replicate_evaluations.md
index 898accc2..bb7146ea 100644
--- a/docs/user-guide/advanced/replicate_evaluations.md
+++ b/docs/user-guide/advanced/replicate_evaluations.md
@@ -8,21 +8,11 @@ Make sure to replace `` in the commands below with `bach`, `crc`, `mhist`
## DINO ViT-S16 (random weights)
-Evaluating the backbone with randomly initialized weights serves as a baseline to compare the pretrained FMs
-to an FM that produces embeddings without any prior learning on image tasks. To evaluate, run:
+Evaluating the backbone with randomly initialized weights serves as a baseline to compare the pretrained FMs to an FM that produces embeddings without any prior learning on image tasks. To evaluate, run:
```
-# set environment variables:
-export PRETRAINED=false
-export EMBEDDINGS_ROOT="./data/embeddings/dino_vits16_random"
-export REPO_OR_DIR=facebookresearch/dino:main
-export DINO_BACKBONE=dino_vits16
-export CHECKPOINT_PATH=null
-export IN_FEATURES=384
-export NORMALIZE_MEAN=[0.485,0.456,0.406]
-export NORMALIZE_STD=[0.229,0.224,0.225]
-
-# run eva:
+PRETRAINED=false \
+EMBEDDINGS_ROOT="./data/embeddings/dino_vits16_random" \
eva predict_fit --config configs/vision/dino_vit/offline/.yaml
```
@@ -31,17 +21,7 @@ eva predict_fit --config configs/vision/dino_vit/offline/.yaml
The next baseline model, uses a pretrained ViT-S16 backbone with ImageNet weights. To evaluate, run:
```
-# set environment variables:
-export PRETRAINED=true
-export EMBEDDINGS_ROOT="./data/embeddings/dino_vits16_imagenet"
-export REPO_OR_DIR=facebookresearch/dino:main
-export DINO_BACKBONE=dino_vits16
-export CHECKPOINT_PATH=null
-export IN_FEATURES=384
-export NORMALIZE_MEAN=[0.485,0.456,0.406]
-export NORMALIZE_STD=[0.229,0.224,0.225]
-
-# run eva:
+EMBEDDINGS_ROOT="./data/embeddings/dino_vits16_imagenet" \
eva predict_fit --config configs/vision/dino_vit/offline/.yaml
```
@@ -49,17 +29,9 @@ eva predict_fit --config configs/vision/dino_vit/offline/.yaml
To evaluate performance on the larger ViT-B8 backbone pretrained on ImageNet, run:
```
-# set environment variables:
-export PRETRAINED=true
-export EMBEDDINGS_ROOT="./data/embeddings/dino_vitb8_imagenet"
-export REPO_OR_DIR=facebookresearch/dino:main
-export DINO_BACKBONE=dino_vitb8
-export CHECKPOINT_PATH=null
-export IN_FEATURES=384
-export NORMALIZE_MEAN=[0.485,0.456,0.406]
-export NORMALIZE_STD=[0.229,0.224,0.225]
-
-# run eva:
+EMBEDDINGS_ROOT="./data/embeddings/dino_vitb8_imagenet" \
+DINO_BACKBONE=dino_vitb8 \
+IN_FEATURES=768 \
eva predict_fit --config configs/vision/dino_vit/offline/.yaml
```
@@ -69,17 +41,11 @@ eva predict_fit --config configs/vision/dino_vit/offline/.yaml
on [GitHub](https://github.com/lunit-io/benchmark-ssl-pathology/releases/). To evaluate, run:
```
-# set environment variables:
-export PRETRAINED=false
-export EMBEDDINGS_ROOT="./data/embeddings/dino_vits16_lunit"
-export REPO_OR_DIR=facebookresearch/dino:main
-export DINO_BACKBONE=dino_vits16
-export CHECKPOINT_PATH="https://github.com/lunit-io/benchmark-ssl-pathology/releases/download/pretrained-weights/dino_vit_small_patch16_ep200.torch"
-export IN_FEATURES=384
-export NORMALIZE_MEAN=[0.70322989,0.53606487,0.66096631]
-export NORMALIZE_STD=[0.21716536,0.26081574,0.20723464]
-
-# run eva:
+PRETRAINED=false \
+EMBEDDINGS_ROOT="./data/embeddings/dino_vits16_lunit" \
+CHECKPOINT_PATH="https://github.com/lunit-io/benchmark-ssl-pathology/releases/download/pretrained-weights/dino_vit_small_patch16_ep200.torch" \
+NORMALIZE_MEAN=[0.70322989,0.53606487,0.66096631] \
+NORMALIZE_STD=[0.21716536,0.26081574,0.20723464] \
eva predict_fit --config configs/vision/dino_vit/offline/.yaml
```
@@ -89,14 +55,11 @@ eva predict_fit --config configs/vision/dino_vit/offline/.yaml
[HuggingFace](https://huggingface.co/owkin/phikon). To evaluate, run:
```
-# set environment variables:
-export EMBEDDINGS_ROOT="./data/embeddings/dino_vitb16_owkin"
-
-# run eva:
+EMBEDDINGS_ROOT="./data/embeddings/dino_vitb16_owkin" \
eva predict_fit --config configs/vision/owkin/phikon/offline/.yaml
```
-Note: since ***eva*** provides the config files to evaluate tasks with the Phikon FM in
+Note: since *eva* provides the config files to evaluate tasks with the Phikon FM in
"configs/vision/owkin/phikon/offline", it is not necessary to set the environment variables needed for
the runs above.
@@ -106,17 +69,11 @@ To evaluate [kaiko.ai's](https://www.kaiko.ai/) FM with DINO ViT-S16 backbone, p
on [GitHub](https://github.com/lunit-io/benchmark-ssl-pathology/releases/), run:
```
-# set environment variables:
-export PRETRAINED=false
-export EMBEDDINGS_ROOT="./data/embeddings/dino_vits16_kaiko"
-export REPO_OR_DIR=facebookresearch/dino:main
-export DINO_BACKBONE=dino_vits16
-export CHECKPOINT_PATH=[TBD*]
-export IN_FEATURES=384
-export NORMALIZE_MEAN=[0.5,0.5,0.5]
-export NORMALIZE_STD=[0.5,0.5,0.5]
-
-# run eva:
+PRETRAINED=false \
+EMBEDDINGS_ROOT="./data/embeddings/dino_vits16_kaiko" \
+CHECKPOINT_PATH=[TBD*] \
+NORMALIZE_MEAN=[0.5,0.5,0.5] \
+NORMALIZE_STD=[0.5,0.5,0.5] \
eva predict_fit --config configs/vision/dino_vit/offline/.yaml
```
@@ -128,17 +85,12 @@ To evaluate [kaiko.ai's](https://www.kaiko.ai/) FM with DINO ViT-S8 backbone, pr
on [GitHub](https://github.com/lunit-io/benchmark-ssl-pathology/releases/), run:
```
-# set environment variables:
-export PRETRAINED=false
-export EMBEDDINGS_ROOT="./data/embeddings/dino_vits8_kaiko"
-export REPO_OR_DIR=facebookresearch/dino:main
-export DINO_BACKBONE=dino_vits8
-export CHECKPOINT_PATH=[TBD*]
-export IN_FEATURES=384
-export NORMALIZE_MEAN=[0.5,0.5,0.5]
-export NORMALIZE_STD=[0.5,0.5,0.5]
-
-# run eva:
+PRETRAINED=false \
+EMBEDDINGS_ROOT="./data/embeddings/dino_vits8_kaiko" \
+DINO_BACKBONE=dino_vits8 \
+CHECKPOINT_PATH=[TBD*] \
+NORMALIZE_MEAN=[0.5,0.5,0.5] \
+NORMALIZE_STD=[0.5,0.5,0.5] \
eva predict_fit --config configs/vision/dino_vit/offline/.yaml
```
@@ -150,17 +102,13 @@ To evaluate [kaiko.ai's](https://www.kaiko.ai/) FM with the larger DINO ViT-B16
run:
```
-# set environment variables:
-export PRETRAINED=false
-export EMBEDDINGS_ROOT="./data/embeddings/dino_vitb16_kaiko"
-export REPO_OR_DIR=facebookresearch/dino:main
-export DINO_BACKBONE=dino_vitb16
-export CHECKPOINT_PATH=[TBD*]
-export IN_FEATURES=768
-export NORMALIZE_MEAN=[0.5,0.5,0.5]
-export NORMALIZE_STD=[0.5,0.5,0.5]
-
-# run eva:
+PRETRAINED=false \
+EMBEDDINGS_ROOT="./data/embeddings/dino_vitb16_kaiko" \
+DINO_BACKBONE=dino_vitb16 \
+CHECKPOINT_PATH=[TBD*] \
+IN_FEATURES=768 \
+NORMALIZE_MEAN=[0.5,0.5,0.5] \
+NORMALIZE_STD=[0.5,0.5,0.5] \
eva predict_fit --config configs/vision/dino_vit/offline/.yaml
```
@@ -172,17 +120,13 @@ To evaluate [kaiko.ai's](https://www.kaiko.ai/) FM with the larger DINO ViT-B8 b
run:
```
-# set environment variables:
-export PRETRAINED=false
-export EMBEDDINGS_ROOT="./data/embeddings/dino_vitb8_kaiko"
-export REPO_OR_DIR=facebookresearch/dino:main
-export DINO_BACKBONE=dino_vitb8
-export CHECKPOINT_PATH=[TBD*]
-export IN_FEATURES=768
-export NORMALIZE_MEAN=[0.5,0.5,0.5]
-export NORMALIZE_STD=[0.5,0.5,0.5]
-
-# run eva:
+PRETRAINED=false \
+EMBEDDINGS_ROOT="./data/embeddings/dino_vitb8_kaiko" \
+DINO_BACKBONE=dino_vitb8 \
+CHECKPOINT_PATH=[TBD*] \
+IN_FEATURES=768 \
+NORMALIZE_MEAN=[0.5,0.5,0.5] \
+NORMALIZE_STD=[0.5,0.5,0.5] \
eva predict_fit --config configs/vision/dino_vit/offline/.yaml
```
@@ -194,18 +138,15 @@ To evaluate [kaiko.ai's](https://www.kaiko.ai/) FM with the larger DINOv2 ViT-L1
run:
```
-# set environment variables:
-export PRETRAINED=false
-export EMBEDDINGS_ROOT="./data/embeddings/dinov2_vitl14_kaiko"
-export REPO_OR_DIR=facebookresearch/dinov2:main
-export DINO_BACKBONE=dinov2_vitl14_reg
-export FORCE_RELOAD=true
-export CHECKPOINT_PATH=[TBD*]
-export IN_FEATURES=1024
-export NORMALIZE_MEAN=[0.5,0.5,0.5]
-export NORMALIZE_STD=[0.5,0.5,0.5]
-
-# run eva:
+PRETRAINED=false \
+EMBEDDINGS_ROOT="./data/embeddings/dinov2_vitl14_kaiko" \
+REPO_OR_DIR=facebookresearch/dinov2:main \
+DINO_BACKBONE=dinov2_vitl14_reg \
+FORCE_RELOAD=true \
+CHECKPOINT_PATH=[TBD*] \
+IN_FEATURES=1024 \
+NORMALIZE_MEAN=[0.5,0.5,0.5] \
+NORMALIZE_STD=[0.5,0.5,0.5] \
eva predict_fit --config configs/vision/dino_vit/offline/.yaml
```
diff --git a/docs/user-guide/getting-started/how_to_use.md b/docs/user-guide/getting-started/how_to_use.md
index 6fca671e..919aa78d 100644
--- a/docs/user-guide/getting-started/how_to_use.md
+++ b/docs/user-guide/getting-started/how_to_use.md
@@ -34,7 +34,7 @@ The setup for an *eva* run is provided in a `.yaml` config file which is defined
A config file specifies the setup for the *trainer* (including callback for the model backbone), the *model* (setup of the trainable decoder) and *data* module.
-To get a better understanding, inspect some of the provided [config files](https://github.com/kaiko-ai/eva/tree/main/configs/vision) (which you will download if you run the tutorials).
+The config files for the datasets and models that _eva_ supports out of the box, you can find on [GitHub](https://github.com/kaiko-ai/eva/tree/main/configs). We recommend that you inspect some of them to get a better understanding of their structure and content.
### Environment variables
diff --git a/docs/user-guide/getting-started/installation.md b/docs/user-guide/getting-started/installation.md
index 666b4ff9..b4df5814 100644
--- a/docs/user-guide/getting-started/installation.md
+++ b/docs/user-guide/getting-started/installation.md
@@ -8,18 +8,9 @@
- Install *eva* and the *eva-vision* package with:
```
-pip install --index-url https://nexus.infra.prd.kaiko.ai/repository/python-all/simple 'kaiko-eva[vision]'
+pip install "kaiko-eva[vision]"
```
-- To be able to use the existing configs, download them into directory where you installed *eva*. You can get them from our blob storage with:
-
-```
-azcopy copy https://kaiko.blob.core.windows.net/long-term-experimental/eva/configs . --recursive=true
-```
-
-(Alternatively you can also download them from the [*eva* GitHub repo](https://github.com/kaiko-ai/eva/tree/main))
-
-
## Run *eva*
Now you are all setup and you can start running *eva* with:
diff --git a/docs/user-guide/tutorials/evaluate_resnet.md b/docs/user-guide/tutorials/evaluate_resnet.md
index d1765385..9937b328 100644
--- a/docs/user-guide/tutorials/evaluate_resnet.md
+++ b/docs/user-guide/tutorials/evaluate_resnet.md
@@ -2,7 +2,7 @@
If you read [How to use eva](../getting-started/how_to_use.md) and followed the Tutorials to this point, you might ask yourself why you would not always use the *offline* workflow to run a complete evaluation. An *offline*-run stores the computed embeddings and runs faster than the *online*-workflow which computes a backbone-forward pass in every epoch.
-One use case for the *online*-workflow is the evaluation of a supervised ML model that does not rely on an backbone/head architecture. To demonstrate this, let's train a ResNet 18 from [Pytoch Image Models (timm)](https://timm.fast.ai/).
+One use case for the *online*-workflow is the evaluation of a supervised ML model that does not rely on a backbone/head architecture. To demonstrate this, let's train a ResNet 18 from [PyTorch Image Models (timm)](https://timm.fast.ai/).
To do this we need to create a new config-file:
@@ -25,14 +25,11 @@ Now let's adapt the new `bach.yaml`-config to the new model:
drop_rate: 0.0
pretrained: false
```
-To reduce training time, let's overwrite some of the default parameters. In the terminal where you run *eva*, set:
-```
-export OUTPUT_ROOT=logs/resnet/bach
-export MAX_STEPS=50
-export LR_VALUE=0.01
-```
-Now train and evaluate the model by running:
+To reduce training time, let's overwrite some of the default parameters. Run the training & evaluation with:
```
+OUTPUT_ROOT=logs/resnet/bach \
+MAX_STEPS=50 \
+LR_VALUE=0.01 \
eva fit --config configs/vision/resnet18/bach.yaml
```
Once the run is complete, take a look at the results in `logs/resnet/bach//results.json` and check out the tensorboard with `tensorboard --logdir logs/resnet/bach`. How does the performance compare to the results observed in the previous tutorials?
diff --git a/docs/user-guide/tutorials/offline_vs_online.md b/docs/user-guide/tutorials/offline_vs_online.md
index a4d75d8c..4ee81115 100644
--- a/docs/user-guide/tutorials/offline_vs_online.md
+++ b/docs/user-guide/tutorials/offline_vs_online.md
@@ -3,10 +3,11 @@
In this tutorial we run *eva* with the three subcommands `predict`, `fit` and `predict_fit`, and take a look at the difference between *offline* and *online* workflows.
### Before you start
+If you haven't downloaded the config files yet, please download them from [GitHub](https://github.com/kaiko-ai/eva/tree/main/configs).
For this tutorial we use the [BACH](../../datasets/bach.md) classification task which is available on [Zenodo](https://zenodo.org/records/3632035) and is distributed under [*Attribution-NonCommercial-ShareAlike 4.0 International*](https://creativecommons.org/licenses/by-nc-nd/4.0/legalcode) license.
-If you have not yet downloaded the BACH data to your machine, open `configs/vision/dino_vit/offline/bach.yaml` and enable automatic download by setting: `download: true`.
+To let **eva** automatically handle the dataset download, you can open `configs/vision/dino_vit/offline/bach.yaml` and set `download: true`. Before doing so, please make sure that your use case is compliant with the dataset license.
## *Offline* evaluations
@@ -15,10 +16,10 @@ If you have not yet downloaded the BACH data to your machine, open `configs/visi
First, let's use the `predict`-command to download the data and compute embeddings. In this example we use a randomly initialized `dino_vits16` as backbone.
Open a terminal in the folder where you installed *eva* and run:
-```
-export PRETRAINED=false
-export EMBEDDINGS_ROOT=./data/embeddings/dino_vits16_random
+```
+PRETRAINED=false \
+EMBEDDINGS_ROOT=./data/embeddings/dino_vits16_random \
eva predict --config configs/vision/dino_vit/offline/bach.yaml
```
@@ -37,14 +38,12 @@ Once the session is complete, verify that:
Now we can use the `fit`-command to evaluate the FM on the precomputed embeddings.
-To ensure a quick run for the purpose of this exercise, let's overwrite some of the default parameters. In the terminal where you run *eva*, set:
-```
-export MAX_STEPS=20
-export LR_VALUE=0.1
-```
+To ensure a quick run for the purpose of this exercise, we overwrite some of the default parameters. Run *eva* to fit the decoder classifier with:
-Now fit the decoder classifier, by running:
```
+N_RUNS=2 \
+MAX_STEPS=20 \
+LR_VALUE=0.1 \
eva fit --config configs/vision/dino_vit/offline/bach.yaml
```
@@ -67,13 +66,11 @@ With the `predict_fit`-command, the two steps above can be executed with one com
Go back to the terminal and execute:
```
-export N_RUNS=1
-export MAX_STEPS=20
-export BATCH_SIZE=256
-export LR_VALUE=0.1
-export PRETRAINED=true
-export EMBEDDINGS_ROOT=./data/embeddings/dino_vits16_pretrained
-
+N_RUNS=1 \
+MAX_STEPS=20 \
+LR_VALUE=0.1 \
+PRETRAINED=true \
+EMBEDDINGS_ROOT=./data/embeddings/dino_vits16_pretrained \
eva predict_fit --config configs/vision/dino_vit/offline/bach.yaml
```
@@ -87,12 +84,10 @@ As in *Step 3* above, we again use a `dino_vits16` pretrained from ImageNet.
Run a complete online workflow with the following command:
```
-export N_RUNS=1
-export MAX_STEPS=20
-export BATCH_SIZE=256
-export LR_VALUE=0.1
-export PRETRAINED=true
-
+N_RUNS=1 \
+MAX_STEPS=20 \
+LR_VALUE=0.1 \
+PRETRAINED=true \
eva fit --config configs/vision/dino_vit/online/bach.yaml
```
diff --git a/mkdocs.yml b/mkdocs.yml
index e50464e5..53ab351e 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -6,9 +6,13 @@ theme:
features:
- navigation.tabs
- navigation.indexes
+ - navigation.expand
- search.suggest
- search.highlight
- content.code.copy
+ icon:
+ repo: fontawesome/brands/github-alt
+ favicon: images/eva-stripes.png
logo: images/eva-stripes.png
palette:
- media: "(prefers-color-scheme: dark)"
@@ -27,6 +31,7 @@ theme:
name: Switch to dark mode
plugins:
search: null
+ mike: null
mkdocstrings:
handlers:
python:
@@ -42,6 +47,12 @@ plugins:
show_root_heading: true
filters:
- '!^_'
+extra:
+ version:
+ provider: mike
+ default:
+ - latest
+ - dev
markdown_extensions:
- toc:
toc_depth: 2
@@ -53,7 +64,7 @@ markdown_extensions:
- pymdownx.snippets
- pymdownx.superfences
nav:
- - eva: index.md
+ - Introduction: index.md
- User Guide:
- user-guide/index.md
- Getting started:
@@ -65,6 +76,15 @@ nav:
- Advanced user guide:
- user-guide/advanced/replicate_evaluations.md
- user-guide/advanced/model_wrappers.md
+ - Datasets:
+ - datasets/index.md
+ - WSI-patches:
+ - BACH: datasets/bach.md
+ - CRC: datasets/crc.md
+ - MHIST: datasets/mhist.md
+ - PatchCamelyon: datasets/patch_camelyon.md
+ - Radiology:
+ - TotalSegmentator: datasets/total_segmentator.md
- Reference API:
- reference/index.md
- Core:
@@ -99,10 +119,3 @@ nav:
- reference/vision/models/networks.md
- Utils:
- IO: reference/vision/utils/io.md
- - Datasets:
- - datasets/index.md
- - BACH: datasets/bach.md
- - CRC: datasets/crc.md
- - MHIST: datasets/mhist.md
- - PatchCamelyon: datasets/patch_camelyon.md
- - TotalSegmentator: datasets/total_segmentator.md
diff --git a/noxfile.py b/noxfile.py
index 4d09dc90..2a60bada 100644
--- a/noxfile.py
+++ b/noxfile.py
@@ -144,7 +144,7 @@ def docs(session: nox.Session) -> None:
"""Builds and deploys the code documentation."""
args = session.posargs or []
session.run_always("pdm", "install", "--group", "docs", external=True)
- session.run("pdm", "run", "mkdocs", *args)
+ session.run("pdm", "run", "mike", *args)
@nox.session
diff --git a/pdm.lock b/pdm.lock
index 6cd99ba3..47325ac6 100644
--- a/pdm.lock
+++ b/pdm.lock
@@ -683,13 +683,13 @@ files = [
[[package]]
name = "importlib-resources"
-version = "6.3.1"
+version = "6.3.2"
requires_python = ">=3.8"
summary = "Read resources from Python packages"
groups = ["dev", "docs"]
files = [
- {file = "importlib_resources-6.3.1-py3-none-any.whl", hash = "sha256:4811639ca7fa830abdb8e9ca0a104dc6ad13de691d9fe0d3173a71304f068159"},
- {file = "importlib_resources-6.3.1.tar.gz", hash = "sha256:29a3d16556e330c3c8fb8202118c5ff41241cc34cbfb25989bbad226d99b7995"},
+ {file = "importlib_resources-6.3.2-py3-none-any.whl", hash = "sha256:f41f4098b16cd140a97d256137cfd943d958219007990b2afb00439fc623f580"},
+ {file = "importlib_resources-6.3.2.tar.gz", hash = "sha256:963eb79649252b0160c1afcfe5a1d3fe3ad66edd0a8b114beacffb70c0674223"},
]
[[package]]
diff --git a/pyproject.toml b/pyproject.toml
index 2149c1c4..db948c73 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -6,8 +6,21 @@ build-backend = "pdm.backend"
[project]
name = "kaiko-eva"
-version = "0.0.0.dev5"
+version = "0.0.0.dev8"
description = "Evaluation Framework for oncology foundation models."
+keywords = [
+ "machine-learning",
+ "evaluation-framework",
+ "oncology",
+ "foundation-models",
+]
+classifiers = [
+ "Topic :: Software Development :: Build Tools",
+ "Programming Language :: Python :: 3",
+ "Programming Language :: Python :: 3.10",
+ "Programming Language :: Python :: 3.11",
+ "Programming Language :: Python :: 3.12",
+]
readme = "README.md"
authors = [
{ name = "Ioannis Gatopoulos", email = "ioannis@kaiko.ai" },
@@ -32,6 +45,11 @@ dependencies = [
"toolz>=0.12.1",
]
+[project.urls]
+Homepage = "https://kaiko-ai.github.io/eva/dev/"
+Repository = "https://github.com/kaiko-ai/eva"
+Documentation = "https://kaiko-ai.github.io/eva/dev/"
+
[project.license]
file = "LICENSE"
diff --git a/src/eva/core/data/datamodules/call.py b/src/eva/core/data/datamodules/call.py
index 0915cc7b..0b002b4c 100644
--- a/src/eva/core/data/datamodules/call.py
+++ b/src/eva/core/data/datamodules/call.py
@@ -19,7 +19,7 @@ def call_method_if_exists(objects: Iterable[Any], /, method: str) -> None:
def _recursive_iter(objects: Iterable[Any], /) -> Iterable[datasets_lib.TorchDataset]:
- """Iterates thought an iterable of objects and their respective iterable values.
+ """Iterates through an iterable of objects and their respective iterable values.
Args:
objects: The objects to iterate from.
diff --git a/src/eva/core/trainers/_logging.py b/src/eva/core/trainers/_logging.py
index 75334ed7..2b59139f 100644
--- a/src/eva/core/trainers/_logging.py
+++ b/src/eva/core/trainers/_logging.py
@@ -11,8 +11,9 @@
def generate_session_id() -> str:
"""Generates and returns a unique string ID of an experiment.
- The ID is composed of the run timestamp and a its config hash. If the
- configuration hash is an empty string, it will use only the timestamp.
+ The ID is composed of the run timestamp and a hash based on th used
+ config. If the configuration hash is an empty string, it will use
+ only the timestamp.
"""
timestamp = _generate_timestamp_hash()
config_hash = _generate_config_hash()
@@ -34,8 +35,8 @@ def _generate_config_hash(max_hash_len: int = 8) -> str:
config_path = _fetch_config_path()
if config_path is None:
logger.warning(
- "No or multiple configuration file found from command line arguments. "
- "No configuration hash code will created for this experiment."
+ "No or multiple configuration files found from command line arguments. "
+ "No configuration hash code will be created for this experiment."
)
return ""
diff --git a/src/eva/core/trainers/trainer.py b/src/eva/core/trainers/trainer.py
index 7b50d51a..25877fc9 100644
--- a/src/eva/core/trainers/trainer.py
+++ b/src/eva/core/trainers/trainer.py
@@ -78,7 +78,7 @@ def run_evaluation_session(
model: modules.ModelModule,
datamodule: datamodules.DataModule,
) -> None:
- """Runs a evaluation session out-of-place.
+ """Runs an evaluation session out-of-place.
It performs an evaluation run (fit and evaluate) the model
`self._n_run` times. Note that the input `base_model` would
diff --git a/src/eva/vision/__init__.py b/src/eva/vision/__init__.py
index 80aa5cb6..0e4bd4ca 100644
--- a/src/eva/vision/__init__.py
+++ b/src/eva/vision/__init__.py
@@ -7,7 +7,7 @@
msg = (
"eva vision requirements are not installed.\n\n"
"Please pip install as follows:\n"
- ' python -m pip install "eva[vision]" --upgrade'
+ ' python -m pip install "kaiko-eva[vision]" --upgrade'
)
raise ImportError(str(e) + "\n\n" + msg) from e
diff --git a/src/eva/vision/data/datasets/_validators.py b/src/eva/vision/data/datasets/_validators.py
index ce0755e6..9989bc45 100644
--- a/src/eva/vision/data/datasets/_validators.py
+++ b/src/eva/vision/data/datasets/_validators.py
@@ -1,5 +1,7 @@
"""Dataset validation related functions."""
+import os
+
from typing_extensions import List, Tuple
from eva.vision.data.datasets import vision
@@ -18,7 +20,7 @@ def check_dataset_integrity(
"""Verifies the datasets integrity.
Raise:
- ValuesError: If the input dataset's values do not
+ ValueError: If the input dataset's values do not
match the expected ones.
"""
if len(dataset) != length:
@@ -42,3 +44,16 @@ def check_dataset_integrity(
f"({(dataset_classes[0], dataset_classes[-1])}) does not match the expected "
f"ones ({first_and_last_labels}). {_SUFFIX_ERROR_MESSAGE}"
)
+
+
+def check_dataset_exists(dataset_dir: str, download_available: bool) -> None:
+ """Verifies that the dataset folder exists.
+
+ Raise:
+ FileNotFoundError: If the dataset folder does not exist.
+ """
+ if not os.path.isdir(dataset_dir):
+ error_message = f"Dataset not found at '{dataset_dir}'."
+ if download_available:
+ error_message += " You can set `download=True` to download the dataset automatically."
+ raise FileNotFoundError(error_message)
diff --git a/src/eva/vision/data/datasets/classification/bach.py b/src/eva/vision/data/datasets/classification/bach.py
index c7900005..935ab609 100644
--- a/src/eva/vision/data/datasets/classification/bach.py
+++ b/src/eva/vision/data/datasets/classification/bach.py
@@ -96,24 +96,25 @@ def class_to_idx(self) -> Dict[str, int]:
return {"Benign": 0, "InSitu": 1, "Invasive": 2, "Normal": 3}
@property
- def dataset_path(self) -> str:
+ def _dataset_path(self) -> str:
"""Returns the path of the image data of the dataset."""
return os.path.join(self._root, "ICIAR2018_BACH_Challenge", "Photos")
@override
def filename(self, index: int) -> str:
image_path, _ = self._samples[self._indices[index]]
- return os.path.relpath(image_path, self.dataset_path)
+ return os.path.relpath(image_path, self._dataset_path)
@override
def prepare_data(self) -> None:
if self._download:
self._download_dataset()
+ _validators.check_dataset_exists(self._root, True)
@override
def configure(self) -> None:
self._samples = folder.make_dataset(
- directory=self.dataset_path,
+ directory=self._dataset_path,
class_to_idx=self.class_to_idx,
extensions=(".tif"),
)
@@ -145,7 +146,7 @@ def __len__(self) -> int:
def _download_dataset(self) -> None:
"""Downloads the dataset."""
for resource in self._resources:
- if os.path.isdir(self.dataset_path):
+ if os.path.isdir(self._dataset_path):
continue
self._print_license()
diff --git a/src/eva/vision/data/datasets/classification/crc.py b/src/eva/vision/data/datasets/classification/crc.py
index 4717e0be..5c661d45 100644
--- a/src/eva/vision/data/datasets/classification/crc.py
+++ b/src/eva/vision/data/datasets/classification/crc.py
@@ -95,12 +95,13 @@ def class_to_idx(self) -> Dict[str, int]:
@override
def filename(self, index: int) -> str:
image_path, *_ = self._samples[index]
- return os.path.relpath(image_path, self._dataset_dir)
+ return os.path.relpath(image_path, self._dataset_path)
@override
def prepare_data(self) -> None:
if self._download:
self._download_dataset()
+ _validators.check_dataset_exists(self._root, True)
@override
def configure(self) -> None:
@@ -135,7 +136,7 @@ def __len__(self) -> int:
return len(self._samples)
@property
- def _dataset_dir(self) -> str:
+ def _dataset_path(self) -> str:
"""Returns the full path of dataset directory."""
dataset_dirs = {
"train": os.path.join(self._root, "NCT-CRC-HE-100K"),
@@ -150,7 +151,7 @@ def _dataset_dir(self) -> str:
def _make_dataset(self) -> List[Tuple[str, int]]:
"""Builds the dataset for the specified split."""
dataset = folder.make_dataset(
- directory=self._dataset_dir,
+ directory=self._dataset_path,
class_to_idx=self.class_to_idx,
extensions=(".tif"),
)
diff --git a/src/eva/vision/data/datasets/classification/mhist.py b/src/eva/vision/data/datasets/classification/mhist.py
index 9f4bda6c..75297183 100644
--- a/src/eva/vision/data/datasets/classification/mhist.py
+++ b/src/eva/vision/data/datasets/classification/mhist.py
@@ -56,6 +56,10 @@ def filename(self, index: int) -> str:
image_filename, _ = self._samples[index]
return image_filename
+ @override
+ def prepare_data(self) -> None:
+ _validators.check_dataset_exists(self._root, False)
+
@override
def configure(self) -> None:
self._samples = self._make_dataset()
diff --git a/src/eva/vision/data/datasets/classification/patch_camelyon.py b/src/eva/vision/data/datasets/classification/patch_camelyon.py
index e18ee36e..e9eaa5f5 100644
--- a/src/eva/vision/data/datasets/classification/patch_camelyon.py
+++ b/src/eva/vision/data/datasets/classification/patch_camelyon.py
@@ -114,6 +114,7 @@ def filename(self, index: int) -> str:
def prepare_data(self) -> None:
if self._download:
self._download_dataset()
+ _validators.check_dataset_exists(self._root, True)
@override
def validate(self) -> None:
diff --git a/src/eva/vision/data/datasets/classification/total_segmentator.py b/src/eva/vision/data/datasets/classification/total_segmentator.py
index f9ddac26..c7c0c88d 100644
--- a/src/eva/vision/data/datasets/classification/total_segmentator.py
+++ b/src/eva/vision/data/datasets/classification/total_segmentator.py
@@ -108,6 +108,7 @@ def filename(self, index: int) -> str:
def prepare_data(self) -> None:
if self._download:
self._download_dataset()
+ _validators.check_dataset_exists(self._root, True)
@override
def configure(self) -> None:
+
-
-
-