From f05e2d627adf580377f9b2ec1f1a548a387f4618 Mon Sep 17 00:00:00 2001 From: Robrecht Cannoodt Date: Thu, 12 Sep 2024 00:03:53 +0200 Subject: [PATCH] update data --- .../data/wf_aggregated_scores.csv | 14 ++++++++++++++ book/workflow_frameworks/data/wf_metadata.csv | 10 +++++++++- book/workflow_frameworks/quality_assessment.qmd | 11 +++++++---- 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/book/workflow_frameworks/data/wf_aggregated_scores.csv b/book/workflow_frameworks/data/wf_aggregated_scores.csv index 76be3c4..10d366f 100644 --- a/book/workflow_frameworks/data/wf_aggregated_scores.csv +++ b/book/workflow_frameworks/data/wf_aggregated_scores.csv @@ -1,36 +1,50 @@ "name","framework","mean_score" "Community","Argo Workflows",0.125 +"Community","Cromwell + WDL",NA "Community","Galaxy",0.75 "Community","Nextflow",1 "Community","Snakemake",0.75 +"Community","Toil + WDL",0.5 "Community","Viash + Nextflow",0.75 "Maintainability","Argo Workflows",0.5 +"Maintainability","Cromwell + WDL",0.7 "Maintainability","Galaxy",0.6 "Maintainability","Nextflow",0.5 "Maintainability","Snakemake",0.7 +"Maintainability","Toil + WDL",0.6 "Maintainability","Viash + Nextflow",1 "Modularity","Argo Workflows",0.8 +"Modularity","Cromwell + WDL",0.7 "Modularity","Galaxy",0.9 "Modularity","Nextflow",0.5 "Modularity","Snakemake",1 +"Modularity","Toil + WDL",0.4 "Modularity","Viash + Nextflow",1 "Polyglot","Argo Workflows",0.666666666666667 +"Polyglot","Cromwell + WDL",0.333333333333333 "Polyglot","Galaxy",0.333333333333333 "Polyglot","Nextflow",0.666666666666667 "Polyglot","Snakemake",0.666666666666667 +"Polyglot","Toil + WDL",0.333333333333333 "Polyglot","Viash + Nextflow",0.666666666666667 "Portability","Argo Workflows",0.5 +"Portability","Cromwell + WDL",0.611111111111111 "Portability","Galaxy",0.666666666666667 "Portability","Nextflow",0.944444444444444 "Portability","Snakemake",0.888888888888889 +"Portability","Toil + WDL",0.472222222222222 "Portability","Viash + Nextflow",0.944444444444444 "Reproducibility","Argo Workflows",0.7 +"Reproducibility","Cromwell + WDL",0.3 "Reproducibility","Galaxy",0.8 "Reproducibility","Nextflow",0.5 "Reproducibility","Snakemake",0.7 +"Reproducibility","Toil + WDL",0.3 "Reproducibility","Viash + Nextflow",0.8 "Scalability","Argo Workflows",0.5 +"Scalability","Cromwell + WDL",0.833333333333333 "Scalability","Galaxy",0.333333333333333 "Scalability","Nextflow",0.833333333333333 "Scalability","Snakemake",0.833333333333333 +"Scalability","Toil + WDL",0.666666666666667 "Scalability","Viash + Nextflow",0.833333333333333 diff --git a/book/workflow_frameworks/data/wf_metadata.csv b/book/workflow_frameworks/data/wf_metadata.csv index eb58bc6..171383a 100644 --- a/book/workflow_frameworks/data/wf_metadata.csv +++ b/book/workflow_frameworks/data/wf_metadata.csv @@ -3,6 +3,14 @@ "Snakemake","J/M","https://snakemake.github.io/","A workflow management system that uses Python-based rules to define dependencies and execute tasks.","https://github.com/snakemake/snakemake","https://snakemake.readthedocs.io/en/stable/","https://doi.org/10.12688/f1000research.29032.2","https://snakemake-wrappers.readthedocs.io/en/stable/","https://github.com/snakemake-workflows https://github.com/snakemake/snakemake-wrappers","snakemake" "Galaxy","R","https://galaxyproject.org/","A web-based platform for creating, running, and sharing data analysis workflows without the need for programming expertise.","https://github.com/galaxyproject/galaxy","https://docs.galaxyproject.org/en/master/","https://doi.org/10.1093/nar/gkae410","https://galaxyproject.org/iuc/","https://github.com/galaxyproject/tools-iuc","galaxy" +"Cromwell + WDL","J","https://cromwell.readthedocs.io/","An execution engine (Cromwell) that uses a workflow description language (WDL) for defining and running scientific workflows, particularly in genomics and bioinformatics.","https://github.com/broadinstitute/cromwell","https://cromwell.readthedocs.io/","https://doi.org/10.7490/f1000research.1114634.1 +https://doi.org/10.4230/OASIcs.SLATE.2014.77","https://broadinstitute.github.io/warp/ +https://biowdl.github.io/tasks/v0.1/index.html","https://github.com/broadinstitute/warp +https://github.com/biowdl/tasks","cromwell_wdl" +"Toil + WDL","J","http://toil.ucsc-cgl.org/","A workflow engine (Toil) that leverages WDL to define and execute scientific workflows with a focus on scalability and fault tolerance.","https://github.com/DataBiosphere/toil","https://toil.readthedocs.io/en/latest/","http://doi.org/10.1038/nbt.3772 +https://doi.org/10.1145/3486897","https://broadinstitute.github.io/warp/ +https://biowdl.github.io/tasks/v0.1/index.html","https://github.com/broadinstitute/warp +https://github.com/biowdl/tasks","toil_wdl" +"Argo Workflows","T","https://argoproj.github.io/workflows/","A Kubernetes-native workflow engine for orchestrating containerized tasks and automating complex workflows.","https://github.com/argoproj/argo-workflows","https://argo-workflows.readthedocs.io",NA,NA,NA,"argo_workflows" "Viash + Nextflow","R","https://viash.io/","A combination of Viash, a tool for defining bioinformatics workflow components, and Nextflow for scalable and reproducible execution.","https://github.com/viash-io/viash","https://viash.io/guide/ https://www.nextflow.io/docs/latest/index.html","https://joss.theoj.org/papers/10.21105/joss.06089","https://web.viash-hub.com/packages","https://github.com/viash-hub","viash_nextflow" -"Argo Workflows","T","https://argoproj.github.io/workflows/","A Kubernetes-native workflow engine for orchestrating containerized tasks and automating complex workflows.","https://github.com/argoproj/argo-workflows","https://argo-workflows.readthedocs.io",NA,NA,NA,"argo_workflows" diff --git a/book/workflow_frameworks/quality_assessment.qmd b/book/workflow_frameworks/quality_assessment.qmd index 5bc1ebd..566a943 100644 --- a/book/workflow_frameworks/quality_assessment.qmd +++ b/book/workflow_frameworks/quality_assessment.qmd @@ -36,7 +36,10 @@ for (i in seq(2, nrow(wf_qc2), by = 1)) { # remove incomplete frameworks and unused questions wf_qc2 <- wf_qc2 |> - select(name, aspect, item, Nextflow, Snakemake, Galaxy, `Viash + Nextflow`, `Argo Workflows`) %>% + select( + name, aspect, item, + Nextflow, Snakemake, Galaxy, `Cromwell + WDL`, + `Toil + WDL`, `Argo Workflows`, `Viash + Nextflow`) %>% filter(!is.na(Nextflow)) %>% filter(item != "The framework provides clear error messages and debugging tools.") @@ -84,9 +87,9 @@ agg <- scores |> summarize(mean_score = mean(score), .groups = "drop") # write to file -write.csv(meta, "notebooks/workflows/data/wf_metadata.csv", row.names = FALSE) -write.csv(criteria, "notebooks/workflows/data/criteria.csv", row.names = FALSE) -write.csv(agg, "notebooks/workflows/data/wf_aggregated_scores.csv", row.names = FALSE) +write.csv(meta, "book/workflow_frameworks/data/wf_metadata.csv", row.names = FALSE) +write.csv(criteria, "book/workflow_frameworks/data/criteria.csv", row.names = FALSE) +write.csv(agg, "book/workflow_frameworks/data/wf_aggregated_scores.csv", row.names = FALSE) ``` Given the abovementioned limitations, we decided to conduct our own quality assessment of workflow frameworks. This assessment is still largely in the works, but we're happy to share the preliminary results with you.