diff --git a/docs/Special/CHIPS_FF.md b/docs/Special/CHIPS_FF.md index d60078489..477c3e458 100644 --- a/docs/Special/CHIPS_FF.md +++ b/docs/Special/CHIPS_FF.md @@ -1,10 +1,24 @@ -# CHIPS FF [WIP] +# CHIPS FF + + The chipsff repository provides a comprehensive framework for performing materials simulations with machine learning force fields (MLFFs). Simulations include structural relaxation, vacancy and surface energy calculations, interface analysis, elastic properties, phonons and thermal properties. The code supports multiple universal MLFFs and integrates with the JARVIS database and the Atomic Simulation Environment (ASE) to facilitate various materials simulations and workflows. + + Reference: [https://github.com/usnistgov/chipsff](https://github.com/usnistgov/chipsff) + + + -
\ No newline at end of file + + +
+ + + + +
NamesLinks
dft_3d_chipsff-ahttps://pages.nist.gov/jarvis_leaderboard/AI/SinglePropertyPrediction/dft_3d_chipsff_a
dft_3d_chipsff-bhttps://pages.nist.gov/jarvis_leaderboard/AI/SinglePropertyPrediction/dft_3d_chipsff_b
dft_3d_chipsff-chttps://pages.nist.gov/jarvis_leaderboard/AI/SinglePropertyPrediction/dft_3d_chipsff_c
dft_3d_chipsff-c11https://pages.nist.gov/jarvis_leaderboard/AI/SinglePropertyPrediction/dft_3d_chipsff_c11
dft_3d_chipsff-c44https://pages.nist.gov/jarvis_leaderboard/AI/SinglePropertyPrediction/dft_3d_chipsff_c44
dft_3d_chipsff-form_enhttps://pages.nist.gov/jarvis_leaderboard/AI/SinglePropertyPrediction/dft_3d_chipsff_form_en
dft_3d_chipsff-kvhttps://pages.nist.gov/jarvis_leaderboard/AI/SinglePropertyPrediction/dft_3d_chipsff_kv
dft_3d_chipsff-surf_enhttps://pages.nist.gov/jarvis_leaderboard/AI/SinglePropertyPrediction/dft_3d_chipsff_surf_en
dft_3d_chipsff-volhttps://pages.nist.gov/jarvis_leaderboard/AI/SinglePropertyPrediction/dft_3d_chipsff_vol
\ No newline at end of file diff --git a/docs/Special/catalysis_mat.md b/docs/Special/catalysis_mat.md index 559e8aa2b..361d598c7 100644 --- a/docs/Special/catalysis_mat.md +++ b/docs/Special/catalysis_mat.md @@ -1,10 +1,24 @@ -# CatalysisMat [WIP] - -Pearson Correlation Coefficient (PCC) values representing the generalizability of GNN models in catalysis-related datasets. The table compares problem-specific GNN models (first eight rows) with generalized or unified models. The values provide insights into the performance and transferability of GNN models across different adsorption environments, illustrating their potential to predict catalytic behavior in both targeted and broad catalytic systems. - -Reference: [https://doi.org/10.26434/chemrxiv-2024-hj733](https://doi.org/10.26434/chemrxiv-2024-hj733) - - +# CatalysisMat + + + +Pearson Correlation Coefficient (PCC) values representing the generalizability of GNN models in catalysis-related datasets. The table compares problem-specific GNN models (first eight rows) with generalized or unified models. The values provide insights into the performance and transferability of GNN models across different adsorption environments, illustrating their potential to predict catalytic behavior in both targeted and broad catalytic systems. + + + +Reference: [https://doi.org/10.26434/chemrxiv-2024-hj733](https://doi.org/10.26434/chemrxiv-2024-hj733) + + + + + - -
+ + + +
+ + + + +
NamesLinks
tinnet_N-eadhttps://pages.nist.gov/jarvis_leaderboard/AI/SinglePropertyPrediction/tinnet_N_ead
tinnet_O-eadhttps://pages.nist.gov/jarvis_leaderboard/AI/SinglePropertyPrediction/tinnet_O_ead
tinnet_OH-eadhttps://pages.nist.gov/jarvis_leaderboard/AI/SinglePropertyPrediction/tinnet_OH_ead
AGRA_O-eadhttps://pages.nist.gov/jarvis_leaderboard/AI/SinglePropertyPrediction/AGRA_O_ead
AGRA_OH-eadhttps://pages.nist.gov/jarvis_leaderboard/AI/SinglePropertyPrediction/AGRA_OH_ead
AGRA_CO-eadhttps://pages.nist.gov/jarvis_leaderboard/AI/SinglePropertyPrediction/AGRA_CO_ead
AGRA_COOH-eadhttps://pages.nist.gov/jarvis_leaderboard/AI/SinglePropertyPrediction/AGRA_COOH_ead
AGRA_CHO-eadhttps://pages.nist.gov/jarvis_leaderboard/AI/SinglePropertyPrediction/AGRA_CHO_ead
\ No newline at end of file diff --git a/jarvis_leaderboard/__init__.py b/jarvis_leaderboard/__init__.py index c6f00a0c6..bf293b6d0 100644 --- a/jarvis_leaderboard/__init__.py +++ b/jarvis_leaderboard/__init__.py @@ -1,2 +1,3 @@ """Version number.""" -__version__ = "2024.10.10" + +__version__ = "2024.10.30" diff --git a/jarvis_leaderboard/specialized_benchmarks.py b/jarvis_leaderboard/specialized_benchmarks.py index 2456339f8..12ad5f652 100644 --- a/jarvis_leaderboard/specialized_benchmarks.py +++ b/jarvis_leaderboard/specialized_benchmarks.py @@ -1,56 +1,96 @@ +import os import glob -from jarvis_leaderboard.rebuild import get_metric_value, get_results -import pprint -from collections import defaultdict -import numpy as np -import plotly.express as px import pandas as pd -import os +import plotly.express as px +from jarvis_leaderboard.rebuild import get_results root_dir = os.path.dirname(os.path.abspath(__file__)) -def catalysis_mat( - benchmarks=[ - "AI-SinglePropertyPrediction-ead-tinnet_N-test-mae.csv.zip", - "AI-SinglePropertyPrediction-ead-tinnet_O-test-mae.csv.zip", - "AI-SinglePropertyPrediction-ead-tinnet_OH-test-mae.csv.zip", - "AI-SinglePropertyPrediction-ead-AGRA_O-test-mae.csv.zip", - "AI-SinglePropertyPrediction-ead-AGRA_OH-test-mae.csv.zip", - "AI-SinglePropertyPrediction-ead-AGRA_CO-test-mae.csv.zip", - "AI-SinglePropertyPrediction-ead-AGRA_COOH-test-mae.csv.zip", - "AI-SinglePropertyPrediction-ead-AGRA_CHO-test-mae.csv.zip", - ], - metric="pearsonr", - replacements=[ - "Out-", - "AI-SinglePropertyPrediction-ead-", - "-test-mae.csv.zip", - ], - md_path="catalysis_mat.md", +def process_benchmarks( + benchmarks, + metric, + replacements, + md_path, width=800, height=800, - desired_order=[], + desired_order=None, + add_links=True, ): - # Find all methods that have contributions for the above benchmarks mem = {} - for i in glob.glob("jarvis_leaderboard/contributions/*/*.csv.zip"): - for j in benchmarks: - if j in i: - tmp = i.split("/")[2] + # Collect contributions + for contrib_path in glob.glob( + "jarvis_leaderboard/contributions/*/*.csv.zip" + ): + for benchmark in benchmarks: + if benchmark in contrib_path: + method = contrib_path.split("/")[2] for r in replacements: - tmp = tmp.replace(r, "") - mem[tmp] = {} - for k in benchmarks: - mem[tmp][k] = [] - - for i in benchmarks: - names, vals = get_results(bench_name=i, metric=metric) - for j, k in zip(names, vals): + method = method.replace(r, "") + mem.setdefault(method, {k: [] for k in benchmarks}) + + # Populate benchmark results + for benchmark in benchmarks: + names, vals = get_results(bench_name=benchmark, metric=metric) + for name, val in zip(names, vals): for r in replacements: - j = j.replace(r, "") - mem[j][i] = float(k) + name = name.replace(r, "") + mem[name][benchmark] = float(val) + detailed_links = ( + '' + + "" + + "" + + "" + + "" + ) + if add_links: + for benchmark in benchmarks: + # print('benchmark',benchmark) + temp = benchmark.split("-") + category = temp[0] + subcat = temp[1] + data_split = temp[4] + prop = temp[2] + dataset = temp[3] + metric = temp[-1] + md_filename = ( + "https://pages.nist.gov/jarvis_leaderboard/" + # "../docs/" + + category + # + method + + "/" + + subcat + # + submod + + "/" + + dataset + + "_" + + prop + ) + # print(md_filename) + md_filename = ( + '' + + md_filename + + "" + ) + # detailed_links.append([dataset+"-"+prop,md_filename]) + elem = ( + " " + + "" + + "" + + "" + ) + detailed_links += elem + detailed_links += "
NamesLinks
" + + dataset + + "-" + + prop + + "" + + md_filename + + "
" + # Prepare DataFrame dat = [] row_names = [] for i, j in mem.items(): @@ -64,32 +104,48 @@ def catalysis_mat( for r in replacements: b = b.replace(r, "") column_names.append(b) - print(column_names) + # print(column_names) + # row_names, dat = zip(*[(k, list(v.values())) for k, v in mem.items()]) + # column_names = [b.replace(r, "") for b in benchmarks for r in replacements] df = pd.DataFrame(dat, index=row_names, columns=column_names) - # print(df) - if not desired_order: - desired_order = column_names - # Reindex the DataFrame to have both rows and columns in the desired order - df_reordered = df.reindex(index=desired_order, columns=column_names) - - # Display the reordered DataFrame - print(df_reordered) - # fig = px.imshow(df, text_auto=True) + + # Reorder DataFrame + # print('column_names',df.columns) + # print('row_names',row_names) + # print('column_names1',column_names) + # print('desired_order',desired_order) + # desired_order = desired_order or column_names + if desired_order: + + df_reordered = df.reindex(index=desired_order, columns=column_names) + else: + + df_reordered = ( + df # .reindex(index=desired_order)#, columns=column_names) + ) + # print('df_reordered',df_reordered) + # Plot and save fig = px.imshow(df_reordered, text_auto=True) - fig.update_layout( - width=width, # Set the width of the figure - height=height, # Set the height of the figure - ) - htm = fig.to_html() - tmp_out = str(fig.to_html(full_html=False, include_plotlyjs="cdn")) - tmp_out = tmp_out.replace("\n", " ").replace(" ", " ") + fig.update_layout(width=width, height=height) + save_md(fig=fig, md_path=md_path, detailed_links=detailed_links) + + +def save_md(fig=None, md_path=None, detailed_links=None): md_path = os.path.join(root_dir, "..", "docs", "Special", md_path) + tmp_out = ( + fig.to_html(full_html=False, include_plotlyjs="cdn") + .replace("\n", " ") + .replace(" ", " ") + ) + with open(md_path, "r") as file: filedata = file.read().splitlines() content = [] for j in filedata: if "" in j: content.append("") + elif "" in j: + content.append("") elif "" in j: content.append("") else: @@ -98,12 +154,39 @@ def catalysis_mat( filedata = filedata.replace( "", "" + tmp_out ) + if detailed_links: + filedata = filedata.replace( + "", "" + detailed_links + ) + with open(md_path, "w") as file: file.write(filedata) + # with open(md_path, "r") as file: + # content = [ + # ( + # line + # if "" not in line + # else f"{tmp_out}" + # ) + # for line in file + # ] + # with open(md_path, "w") as file: + # file.write("\n".join(content)) + -def chips_ff( - benchmarks=[ +if __name__ == "__main__": + catalysis_benchmarks = [ + "AI-SinglePropertyPrediction-ead-tinnet_N-test-mae.csv.zip", + "AI-SinglePropertyPrediction-ead-tinnet_O-test-mae.csv.zip", + "AI-SinglePropertyPrediction-ead-tinnet_OH-test-mae.csv.zip", + "AI-SinglePropertyPrediction-ead-AGRA_O-test-mae.csv.zip", + "AI-SinglePropertyPrediction-ead-AGRA_OH-test-mae.csv.zip", + "AI-SinglePropertyPrediction-ead-AGRA_CO-test-mae.csv.zip", + "AI-SinglePropertyPrediction-ead-AGRA_COOH-test-mae.csv.zip", + "AI-SinglePropertyPrediction-ead-AGRA_CHO-test-mae.csv.zip", + ] + chips_benchmarks = [ "AI-SinglePropertyPrediction-a-dft_3d_chipsff-test-mae.csv.zip", "AI-SinglePropertyPrediction-b-dft_3d_chipsff-test-mae.csv.zip", "AI-SinglePropertyPrediction-c-dft_3d_chipsff-test-mae.csv.zip", @@ -113,91 +196,17 @@ def chips_ff( "AI-SinglePropertyPrediction-kv-dft_3d_chipsff-test-mae.csv.zip", "AI-SinglePropertyPrediction-surf_en-dft_3d_chipsff-test-mae.csv.zip", "AI-SinglePropertyPrediction-vol-dft_3d_chipsff-test-mae.csv.zip", - ], - metric="pearsonr", - replacements=[ - "AI-SinglePropertyPrediction-", - "-test-mae.csv.zip", - ], - md_path="CHIPS_FF.md", - width=800, - height=800, - desired_order=[], -): - # Find all methods that have contributions for the above benchmarks - mem = {} - for i in glob.glob("jarvis_leaderboard/contributions/*/*.csv.zip"): - for j in benchmarks: - if j in i: - tmp = i.split("/")[2] - for r in replacements: - tmp = tmp.replace(r, "") - mem[tmp] = {} - for k in benchmarks: - mem[tmp][k] = [] - - for i in benchmarks: - names, vals = get_results(bench_name=i, metric=metric) - for j, k in zip(names, vals): - for r in replacements: - j = j.replace(r, "") - mem[j][i] = float(k) - # print('mem',mem) - dat = [] - row_names = [] - for i, j in mem.items(): - row_names.append(i) - tmp = [] - for m, n in j.items(): - tmp.append(n) - dat.append(tmp) - column_names = [] - for b in benchmarks: - for r in replacements: - b = b.replace(r, "") - column_names.append(b) - # print("column names",column_names) - df = pd.DataFrame(dat, index=row_names, columns=column_names) - # print('df',df) - if not desired_order: - desired_order = column_names - # Reindex the DataFrame to have both rows and columns in the desired order - df_reordered = df # df.reindex(index=desired_order, columns=column_names) - - # Display the reordered DataFrame - # print(df_reordered) - # fig = px.imshow(df, text_auto=True) - fig = px.imshow(df_reordered, text_auto=True) - fig.update_layout( - width=width, # Set the width of the figure - height=height, # Set the height of the figure - ) - htm = fig.to_html() - tmp_out = str(fig.to_html(full_html=False, include_plotlyjs="cdn")) - tmp_out = tmp_out.replace("\n", " ").replace(" ", " ") - md_path = os.path.join(root_dir, "..", "docs", "Special", md_path) - with open(md_path, "r") as file: - filedata = file.read().splitlines() - content = [] - for j in filedata: - if "" in j: - content.append("") - elif "" in j: - content.append("") - else: - content.append(j) - filedata = "\n".join(content) - filedata = filedata.replace( - "", "" + tmp_out - ) - with open(md_path, "w") as file: - file.write(filedata) - + ] -if __name__ == "__main__": - htm = chips_ff() - # print(htm) - htm = catalysis_mat( + process_benchmarks( + benchmarks=catalysis_benchmarks, + metric="pearsonr", + replacements=[ + "Out-", + "AI-SinglePropertyPrediction-ead-", + "-test-mae.csv.zip", + ], + md_path="catalysis_mat.md", desired_order=[ "tinnet_N", "tinnet_O", @@ -211,5 +220,11 @@ def chips_ff( "CHGNet_pretrained", "MACE_pretrained", "MATGL_pretrained", - ] + ], + ) + process_benchmarks( + benchmarks=chips_benchmarks, + metric="pearsonr", + replacements=["AI-SinglePropertyPrediction-", "-test-mae.csv.zip"], + md_path="CHIPS_FF.md", ) diff --git a/setup.py b/setup.py index 1f843970d..f12a63c43 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name="jarvis_leaderboard", # Replace with your own username - version="2024.10.10", + version="2024.10.30", author="Kamal Choudhary", author_email="kamal.choudhary@nist.gov", description="jarvis_leaderboard", @@ -16,22 +16,13 @@ "scikit-learn>=0.24.1", "pandas>=1.2.4", "rouge>=1.0.1", - "mkdocs>=1.5.2", - "mkdocs-material>=9.0.5", "pydantic>=2.3.0", "markdown>=3.2.1", "plotly", "absl-py==1.4.0", "nltk==3.8.1", - # "evaluate==0.4.0", - # "rouge-score==0.1.2", - # "fsspec==2023.9.0", - # "aiohttp==3.8.5", - # "datasets==2.14.5", - # "alignn>=2022.10.23", - # "flake8>=3.9.1", - # "pycodestyle>=2.7.0", - # "pydocstyle>=6.0.0", + # "mkdocs>=1.5.2", + # "mkdocs-material>=9.0.5", ], scripts=[ "jarvis_leaderboard/jarvis_populate_data.py",