Update main README (#325)

kaiko-ai · Mar 20, 2024 · 6785b83 · 6785b83
1 parent 32a3583
commit 6785b83
Show file tree

Hide file tree

Showing 3 changed files with 122 additions and 36 deletions.
diff --git a/README.md b/README.md
@@ -6,79 +6,147 @@
 
 _Oncology FM Evaluation Framework by kaiko.ai_
 
-
-<a href="https://www.apache.org/licenses/LICENSE-2.0">
-  <img src="https://img.shields.io/badge/License-Apache%202.0-blue?style=flat-square" />
-</a>
-
-<br />
-<br />
+[![PyPI](https://img.shields.io/pypi/v/kaiko-eva.svg?logo=python)](https://pypi.python.org/pypi/kaiko-eva)
+[![CI](https://github.com/kaiko-ai/eva/workflows/CI/badge.svg)](https://github.com/kaiko-ai/eva/actions?query=workflow%3ACI)
+[![license](https://img.shields.io/badge/License-Apache%202.0-blue.svg?labelColor=gray)](https://github.com/kaiko-ai/eva#license)
 
 <p align="center">
   <a href="#installation">Installation</a> •
   <a href="#how-to-use">How To Use</a> •
-  <a href="#datasets">Datasets</a> •
-  <a href="#contributing">Contribute</a>
+  <a href="https://kaiko-ai.github.io/eva/">Documentation</a> •
+  <a href="https://kaiko-ai.github.io/eva/dev/datasets/">Datasets</a> •
+  <a href="#benchmarks">Benchmarks</a> <br>
+  <a href="#contributing">Contribute</a> •
+  <a href="#acknowledgements">Acknowledgements</a>
 </p>
 
 </div>
 
----
+### Introduction
+
+_`eva`_ is an evaluation framework for oncology foundation models (FMs) by [kaiko.ai](https://kaiko.ai/). Check out the [documentation](https://kaiko-ai.github.io/eva/) for more information.
+
+<div align="center">
 
-### _About_
+<img src="./docs/images/eva-process.gif" width="800">
 
-`eva` is [kaiko.ai](https://kaiko.ai/)'s evaluation framework for oncology foundation models (FMs). Check out the documentation (LINK TO BE ADDED) for more information.
+</div>
 
+### Highlights:
+- Easy and reliable benchmark of Oncology FMs
+- Automatic embedding inference and evaluation of a downstream task
+- Native support of popular medical [datasets](https://kaiko-ai.github.io/eva/dev/datasets/) and models
+- Produce statistics over multiple evaluation fits and multiple metrics
 
 ## Installation
 
-*Note: this section will be revised for the public package when publishing eva*
+Simple installation from PyPI:
+```sh
+# to install the core version only
+pip install kaiko-eva
 
-- Create and activate a virtual environment with Python 3.10+
+# to install the expanded `vision` version
+pip install 'kaiko-eva[vision]'
 
-- Install *eva* and the *eva-vision* package with:
+# to install everything
+pip install 'kaiko-eva[all]'
+```
 
+To install the latest version of the `main` branch:
+```sh
+pip install "kaiko-eva[vision] @ git+https://github.com/kaiko-ai/eva.git"
 ```
-pip install 'kaiko-eva[vision]'
+
+You can verify that the installation was successful by executing:
+```sh
+eva --version
 ```
 
-- To be able to use the existing configs, download them from the [*eva* GitHub repo](https://github.com/kaiko-ai/eva/tree/main) and move them to directory where you installed *eva*. 
+## How To Use
 
-### Run *eva*
+_eva_ can be used directly from the terminal as a CLI tool as follows:
+```sh
+eva {fit,predict,predict_fit} --config url/or/path/to/the/config.yaml 
+```
 
-Now you can run a complete *eva* workflow, for example with:
+For example, to perform a downstream evaluation of DINO ViT-S/16 on the BACH dataset with linear probing by first inferring the embeddings and performing 5 sequential fits, execute:
+```sh
+eva predict_fit --config https://raw.githubusercontent.com/kaiko-ai/eva/main/configs/vision/dino_vit/offline/bach.yaml
 ```
-eva fit --config configs/vision/dino_vit/online/bach.yaml 
+
+> [!NOTE] 
+> All the datasets that support automatic download in the repo have by default the option to automatically download set to false. For automatic download you have to manually set download=true.
+
+
+To view all the possibles, execute:
+```sh
+eva --help
 ```
-This will:
 
- - Download and extract the dataset, if it has not been downloaded before.
- - Fit a model consisting of the frozen FM-backbone and a classification head on the train split.
- - Evaluate the trained model on the validation split and report the results.
+For more information, please refer to the [documentation](https://kaiko-ai.github.io/eva/dev/user-guide/tutorials/offline_vs_online/) and [tutorials](https://kaiko-ai.github.io/eva/dev/user-guide/advanced/replicate_evaluations/).
+
+## Benchmarks
+
+In this section you will find model benchmarks which were generated with _eva_.
+
+### Table I: WSI patch-level benchmark
 
-For more information, documentation and tutorials, refer to the documentation (LINK TO BE ADDED).
+<br />
 
-## Datasets
+<div align="center">
 
-The following datasets are supported natively:
+| Model                                            | BACH  | CRC   | MHIST | PCam/val | PCam/test |
+|--------------------------------------------------|-------|-------|-------|----------|-----------|
+| ViT-S/16 _(random)_	<sup>[1]</sup>               | 0.410 | 0.617 | 0.501 | 0.753    | 0.728     |
+| ViT-S/16 _(ImageNet)_ <sup>[1]</sup>             | 0.695 | 0.935 | 0.831 | 0.864    | 0.849     |
+| ViT-B/8 _(ImageNet)_ <sup>[1]</sup>              | 0.797 | 0.943 | 0.828 | 0.903    | 0.893     |
+| DINO<sub>(p=16)</sub> <sup>[2]</sup>             | 0.710 | 0.935 | 0.814 | 0.870    | 0.856     |
+| Phikon <sup>[3]</sup>                            | 0.725 | 0.935 | 0.777 | 0.912    | 0.915     |
+| ViT-S/16 _(kaiko.ai)_ <sup>[4]</sup>             | 0.797 | 0.943 | 0.828 | 0.903    | 0.893     |
+| ViT-S/8 _(kaiko.ai)_ <sup>[4]</sup>              | 0.834 | 0.946 | 0.832 | 0.897    | 0.887     |
+| ViT-B/16 _(kaiko.ai)_	<sup>[4]</sup>             | 0.810 | 0.960 | 0.826 | 0.900    | 0.898     |
+| ViT-B/8 _(kaiko.ai)_ <sup>[4]</sup>              | 0.865 | 0.956 | 0.809 | 0.913    | 0.921     |
+| ViT-L/14 _(kaiko.ai)_ <sup>[4]</sup>             | 0.870 | 0.930 | 0.809 | 0.908    | 0.898     |
+
+_Table I: Linear probing evaluation of FMs on patch-level downstream datasets.<br> We report averaged balanced accuracy
+over 5 runs_, with an average standard deviation of ±0.003.
 
-### Vision
+</div>
 
-#### Patch-level pathology datasets:
-  - [BACH](./docs/datasets/bach.md)
-  - [CRC](./docs/datasets/crc.md)
-  - [MHIST](./docs/datasets/mhist.md)
-  - [PatchCamelyon](./docs/datasets/patch_camelyon.md)
+<br />
 
-#### Radiology datasets:
-  - [TotalSegmentator](./docs/datasets/total_segmentator.md)
+_References_:
+1. _"Emerging properties in self-supervised vision transformers”_
+2. _"Benchmarking self-supervised learning on diverse pathology datasets”_
+3. _"Scaling self-supervised learning for histopathology with masked image modeling”_
+4. _"Towards training Large-Scale Medical Foundation Models: from TCGA to hospital-scale pathology FMs”_
 
 ## Contributing
 
 _eva_ is an open source project and welcomes contributions of all kinds. Please checkout the [developer](./docs/DEVELOPER_GUIDE.md) and [contributing guide](./docs/CONTRIBUTING.md) for help on how to do so.
 
 All contributors must follow the [code of conduct](./docs/CODE_OF_CONDUCT.md).
 
+
+## Acknowledgements
+
+Our codebase is built using multiple opensource contributions
+
+<div align="center">
+
+[![python](https://img.shields.io/badge/-Python-blue?logo=python&logoColor=white)](https://github.com/pre-commit/pre-commit)
+[![pytorch](https://img.shields.io/badge/PyTorch-ee4c2c?logo=pytorch&logoColor=white)](https://pytorch.org/get-started/locally/)
+[![lightning](https://img.shields.io/badge/-⚡️_Lightning-792ee5?logo=pytorchlightning&logoColor=white)](https://pytorchlightning.ai/)<br>
+[![black](https://img.shields.io/badge/Code%20Style-Black-black.svg?labelColor=gray)](https://black.readthedocs.io/en/stable/)
+[![isort](https://img.shields.io/badge/%20imports-isort-%231674b1?style=flat&labelColor=ef8336)](https://pycqa.github.io/isort/)
+[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
+[![Checked with pyright](https://microsoft.github.io/pyright/img/pyright_badge.svg)](https://microsoft.github.io/pyright/)<br>
+[![pdm-managed](https://img.shields.io/badge/pdm-managed-blueviolet)](https://pdm-project.org)
+[![Nox](https://img.shields.io/badge/%F0%9F%A6%8A-Nox-D85E00.svg)](https://github.com/wntrblm/nox)
+[![Built with Material for MkDocs](https://img.shields.io/badge/Material_for_MkDocs-526CFE?logo=MaterialForMkDocs&logoColor=white)](https://squidfunk.github.io/mkdocs-material/)
+
+</div>
+
 ---
 <div align="center">
   <img src="./docs/images/kaiko-logo.png" width="200">

diff --git a/docs/images/eva-process.gif b/docs/images/eva-process.gif
diff --git a/pyproject.toml b/pyproject.toml
@@ -6,8 +6,21 @@ build-backend = "pdm.backend"
 
 [project]
 name = "kaiko-eva"
-version = "0.0.0.dev5"
+version = "0.0.0.dev6"
 description = "Evaluation Framework for oncology foundation models."
+keywords = [
+    "machine-learning",
+    "evaluation-framework",
+    "oncology",
+    "foundation-models",
+]
+classifiers = [
+    "Topic :: Software Development :: Build Tools",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+]
 readme = "README.md"
 authors = [
     { name = "Ioannis Gatopoulos", email = "[email protected]" },
@@ -32,6 +45,11 @@ dependencies = [
     "toolz>=0.12.1",
 ]
 
+[project.urls]
+Homepage = "https://kaiko-ai.github.io/eva/dev/"
+Repository = "https://github.com/kaiko-ai/eva"
+Documentation = "https://kaiko-ai.github.io/eva/dev/"
+
 [project.license]
 file = "LICENSE"