Add bulk EOS and E-V scanning benchmark (#56)

Add energy-volume scans (E-V scans) and equations of state (EOS) from 1,000 WBM subset selected with renormalized probability to balance element distribution.

* add structure db and workflow script

* add bulk eos parquets

* analysis

* fix e-v scanning code, update analysis (it will fail bc it is processing the old data)

* refactor and submit wbm ev scan

* add wbm ev to registry

* fix path mkdir

* fix path mkdir

* convert mattersim numpy float into primitive float

* add cache, fix mattersim

* add wbm ev scan parquets

* analyze EoS and EV scan

* json lfs

* update readme

* update analysis.py and summary tables; add hf page and rank

* delete outdated results and notebook

* update eos result and page

---------

Co-authored-by: matthewkuner <[email protected]>

track parquet with lfs

track ase db

.gitattributes

@@ -1 +1,3 @@
 *.json filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.db filter=lfs diff=lfs merge=lfs -text

.github/README.md

@@ -18,12 +18,12 @@ MLIP Arena leverages modern pythonic workflow orchestrator [Prefect](https://www
 
 ## Announcement
 
-- **[April 8, 2025]** [🎉 **MLIP Arena accepted as an ICLR AI4Mat Spotlight!** 🎉](https://openreview.net/forum?id=ysKfIavYQE#discussion) Huge thanks to all co-authors for their contributions!
+- **[April 8, 2025]** [🎉 **MLIP Arena is accepted as an ICLR AI4Mat Spotlight!** 🎉](https://openreview.net/forum?id=ysKfIavYQE#discussion) Huge thanks to all co-authors for their contributions!
 
 
 ## Installation
 
-### From PyPI (without model running capability)
+### From PyPI (prefect workflow only, without pretrained models)
 
 ```bash
 pip install mlip-arena

examples/eos_bulk/CHGNet.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:68871d694e93a3c3e7e272b9cbd87d3757e3bc689f30f3189db232d76e629c07
+size 429910

examples/eos_bulk/CHGNet_processed.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6fbea63f9035e376bb5ac7db38175102ab3f96a0f8758cc3e9931424f829ac0
+size 357919

examples/eos_bulk/M3GNet.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53dde465b5e10edd677f131f8a531e3dfc36303dd7ec7b9df0060c19847494d9
+size 427419

examples/eos_bulk/M3GNet_processed.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:18ea51bf19c5e011e170a3229bbd63ce675a725c364e0cffb71de95459f8629e
+size 379859

examples/eos_bulk/MACE-MP(M).parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff9769eeb83042129767aeff975eb04dee8efae12e96fbd46cd3039eeda26705
+size 427896

examples/eos_bulk/MACE-MP(M)_processed.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f492125c87400fee013d32904ee97df49a07d321bd393f9335c7bf4258fe159
+size 371004

examples/eos_bulk/MACE-MPA.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53fcd188baddd4d5e797c5aa3de1b4368db711ebd29b7877cfe224856ba9d171
+size 428888

examples/eos_bulk/MACE-MPA_processed.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:34bd9ae08656e374263820774f49e91a964aa8b2aeade4150cf62cfc08bb37f6
+size 365289

examples/eos_bulk/MatterSim.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6717650b97782de6f90e4473075410fe4540279eb39338d2234d3c9399079b3
+size 389586

examples/eos_bulk/MatterSim_processed.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cca22b5db67dae59602adfb8c42a80da90b39c4e95af89ef918813351b422119
+size 320962

examples/eos_bulk/ORBv2.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae13c9af1ae7fafe2a42ed4c47e2ba0f036abfa64a87ca517b92d89c62fcbfd9
+size 427105

examples/eos_bulk/ORBv2_processed.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c90e61645c83f2452bcd90ef9132c46266896217fe1dea9cca8e0d124d73821a
+size 227929

examples/eos_bulk/SevenNet.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64be88ec2632cdabf79daa01acb2cf2ef19fef0557813df5502c4f71ec566f4e
+size 428341

examples/eos_bulk/SevenNet_processed.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5cc03d9af93c001f3fa441b50f058506e13c0aa7cb3d329275e68f5ed80dc3e6
+size 364846

examples/eos_bulk/preprocessing.py

@@ -0,0 +1,12 @@
+import json
+
+from ase.db import connect
+from pymatgen.core import Structure
+
+with open("wbm_structures.json") as f:
+    structs = json.load(f)
+
+with connect("wbm_structures.db") as db:
+    for id, s in structs.items():
+        atoms = Structure.from_dict(s).to_ase_atoms(msonable=False)
+        db.write(atoms, wbm_id=id)

examples/eos_bulk/run.py

@@ -0,0 +1,135 @@
+import functools
+from pathlib import Path
+
+import pandas as pd
+from ase.db import connect
+from dask.distributed import Client
+from dask_jobqueue import SLURMCluster
+from prefect import Task, flow, task
+from prefect.client.schemas.objects import TaskRun
+from prefect.states import State
+from prefect_dask import DaskTaskRunner
+
+from mlip_arena.models import REGISTRY, MLIPEnum
+from mlip_arena.tasks.eos import run as EOS
+from mlip_arena.tasks.optimize import run as OPT
+from mlip_arena.tasks.utils import get_calculator
+
+
+@task
+def load_wbm_structures():
+    """
+    Load the WBM structures from a ASE DB file.
+    """
+    with connect("../wbm_structures.db") as db:
+        for row in db.select():
+            yield row.toatoms(add_additional_information=True)
+
+
+def save_result(
+    tsk: Task,
+    run: TaskRun,
+    state: State,
+    model_name: str,
+    id: str,
+):
+    result = run.state.result()
+
+    assert isinstance(result, dict)
+
+    result["method"] = model_name
+    result["id"] = id
+    result.pop("atoms", None)
+
+    fpath = Path(f"{model_name}")
+    fpath.mkdir(exist_ok=True)
+
+    fpath = fpath / f"{result['id']}.pkl"
+
+    df = pd.DataFrame([result])
+    df.to_pickle(fpath)
+
+
+@task
+def eos_bulk(atoms, model):
+
+    calculator = get_calculator(
+        model
+    )  # avoid sending entire model over prefect and select freer GPU
+
+    result = OPT.with_options(
+        refresh_cache=True,
+    )(
+        atoms,
+        calculator,
+        optimizer="FIRE",
+        criterion=dict(
+            fmax=0.1,
+        ),
+    )
+
+    return EOS.with_options(
+        refresh_cache=True,
+        on_completion=[functools.partial(
+            save_result,
+            model_name=model.name,
+            id=atoms.info["key_value_pairs"]["wbm_id"],
+        )],
+    )(
+        atoms=result["atoms"],
+        calculator=calculator,
+        optimizer="FIRE",
+        npoints=21,
+        max_abs_strain=0.2,
+        concurrent=False
+    )
+
+
+@flow
+def run_all():
+    futures = []
+    for atoms in load_wbm_structures():
+        for model in MLIPEnum:
+            if "eos_bulk" not in REGISTRY[model.name].get("gpu-tasks", []):
+                continue
+            result = eos_bulk.submit(atoms, model)
+            futures.append(result)
+    return [f.result(raise_on_failure=False) for f in futures]
+
+
+nodes_per_alloc = 1
+gpus_per_alloc = 1
+ntasks = 1
+
+cluster_kwargs = dict(
+    cores=4,
+    memory="64 GB",
+    shebang="#!/bin/bash",
+    account="m3828",
+    walltime="00:50:00",
+    job_mem="0",
+    job_script_prologue=[
+        "source ~/.bashrc",
+        "module load python",
+        "source activate /pscratch/sd/c/cyrusyc/.conda/mlip-arena",
+    ],
+    job_directives_skip=["-n", "--cpus-per-task", "-J"],
+    job_extra_directives=[
+        "-J eos_bulk",
+        "-q regular",
+        f"-N {nodes_per_alloc}",
+        "-C gpu",
+        f"-G {gpus_per_alloc}",
+        "--exclusive",
+    ],
+)
+
+cluster = SLURMCluster(**cluster_kwargs)
+print(cluster.job_script())
+cluster.adapt(minimum_jobs=20, maximum_jobs=40)
+client = Client(cluster)
+
+run_all.with_options(
+    task_runner=DaskTaskRunner(address=client.scheduler.address),
+    log_prints=True,
+)()

examples/eos_bulk/summary.csv

@@ -0,0 +1,8 @@
+model,rank,rank-aggregation,energy-diff-flip-times,tortuosity,spearman-compression-energy,spearman-compression-derivative,spearman-tension-energy,missing
+MACE-MPA,1,6,1.0370741482965933,1.005455197941088,-0.9993684338373716,0.9963320580555048,0.993186372745491,2
+MACE-MP(M),2,16,1.042211055276382,1.008986842539345,-0.999329983249581,0.9941160347190496,0.9915857612939804,5
+MatterSim,3,18,1.045135406218656,1.0060900449752808,-0.99734962463147,0.9927904926901917,0.9880977115916667,3
+CHGNet,4,22,1.1053159478435306,1.014753469076796,-0.9964985866690981,0.9929971733381963,0.9866417434120545,3
+SevenNet,5,27,1.1093279839518555,1.0186969977862483,-0.9981277164827815,0.9889121911188109,0.9859580417030127,3
+M3GNet,6,33,1.1748743718592964,1.0175007963267957,-0.9963209989340641,0.9897426526572255,0.9801690217498693,5
+ORBv2,7,42,1.3162134944612287,1.0374718753890275,-0.9918459519667977,0.9701425127407,0.9637462235649547,7

examples/eos_bulk/summary.tex

@@ -0,0 +1,13 @@
+\begin{tabular}{lrrrrrrrl}
+\toprule
+model & rank & rank-aggregation & energy-diff-flip-times & tortuosity & spearman-compression-energy & spearman-compression-derivative & spearman-tension-energy & missing \\
+\midrule
+MACE-MPA & 1 & 6 & 1.037074 & 1.005455 & -0.999368 & 0.996332 & 0.993186 & 2 \\
+MACE-MP(M) & 2 & 16 & 1.042211 & 1.008987 & -0.999330 & 0.994116 & 0.991586 & 5 \\
+MatterSim & 3 & 18 & 1.045135 & 1.006090 & -0.997350 & 0.992790 & 0.988098 & 3 \\
+CHGNet & 4 & 22 & 1.105316 & 1.014753 & -0.996499 & 0.992997 & 0.986642 & 3 \\
+SevenNet & 5 & 27 & 1.109328 & 1.018697 & -0.998128 & 0.988912 & 0.985958 & 3 \\
+M3GNet & 6 & 33 & 1.174874 & 1.017501 & -0.996321 & 0.989743 & 0.980169 & 5 \\
+ORBv2 & 7 & 42 & 1.316213 & 1.037472 & -0.991846 & 0.970143 & 0.963746 & 7 \\
+\bottomrule
+\end{tabular}

examples/wbm_ev/ALIGNN.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b84592b56c667f49e510f382c07f2dd4105df71468c2198c3958b2d0066202b
+size 425244

examples/wbm_ev/ALIGNN_processed.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1d3f3d2992c02464fdc5a4d58ca05de553dffab355b08e99a46d3b6d2495d11
+size 368547

examples/wbm_ev/CHGNet.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06342370be572819441a9c706f3e70555c6ac0bf75d0fdaa35f2f574c9f600cd
+size 424462

examples/wbm_ev/CHGNet_processed.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50bbb0ed4ba3e8af06c2a90e927182132be1f988643dd7b6844d39fe4dd1084c
+size 357683

examples/wbm_ev/M3GNet.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:841aaa082db265939b3a3ada6f0d6901e65cb942277b074ca55cbdd7730dde75
+size 411741

examples/wbm_ev/M3GNet_processed.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:411b2f619314dafb34349a65d985b7b881cb21687ecb9a853da29fc21d6fa714
+size 357786

examples/wbm_ev/MACE-MP(M).parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1551823daef888914de951f2610ee6ffdfd2b0d6f33e1e293614534cdd217196
+size 409083

examples/wbm_ev/MACE-MP(M)_processed.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:877cd9e9407f9402fc01510d051ad19db815e9794c87ee8402f306e2afafd45e
+size 359765

examples/wbm_ev/MACE-MPA.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eedf48b2478811a1dca46eb50d607004dca99f54c81909f331c550317f14cd19
+size 407912

examples/wbm_ev/MACE-MPA_processed.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee5167ef38acf130548c5f5cbf78fedd14b7de7df47c72d771997f4e13302c0b
+size 356642

examples/wbm_ev/MatterSim.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b3d587fd71a817968a513b727a174c353c97552bc7674e5d3a4108e2b6233556
+size 408998

examples/wbm_ev/MatterSim_processed.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:107a30bb5541a861e4141be0db3de210ca2229a22d975f732f107f0e6afdeb0f
+size 356292

examples/wbm_ev/ORBv2.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ee9f2322096fbeb103a85b0735ed0d547f93e34f1c113af3619baf35c7acbc3
+size 415496

examples/wbm_ev/ORBv2_processed.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:01525c5b2e37b8d930c94ee2dc257b8d7c36b24111206ba2efc499a8bc172fcd
+size 357949

examples/wbm_ev/SevenNet.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6850c333c0b754b942efcfac11739ae199a9f3c816da4e0e6b26bc9a037a0524
+size 410197

examples/wbm_ev/SevenNet_processed.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83d9c45216940262575bc7386c62f8b79bb3ac5e63b8ab33ca4307976bb9796f
+size 358345

examples/wbm_ev/eqV2(OMat).parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7521597cd3189c0a3cea18ae98d9310bfc6becd50fca5e6f2c97af5e69b2596d
+size 414251

examples/wbm_ev/eqV2(OMat)_processed.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50f318dce825e939a5c24aa8789acb2952865e0c682462f99cfed204ea9fba64
+size 356693

examples/wbm_ev/run.py

@@ -0,0 +1,163 @@
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+from ase.db import connect
+from dask.distributed import Client
+from dask_jobqueue import SLURMCluster
+from prefect import flow, task
+from prefect.runtime import task_run
+from prefect_dask import DaskTaskRunner
+from prefect.cache_policies import INPUTS, TASK_SOURCE
+
+from mlip_arena.models import REGISTRY, MLIPEnum
+from mlip_arena.tasks.utils import get_calculator
+
+
+@task
+def load_wbm_structures():
+    """
+    Load the WBM structures from an ASE database file.
+    
+    Reads structures from 'wbm_structures.db' and yields them as ASE Atoms objects
+    with additional metadata preserved from the database.
+    
+    Yields:
+        ase.Atoms: Individual atomic structures from the WBM database with preserved
+                  metadata in the .info dictionary.
+    """
+    with connect("../wbm_structures.db") as db:
+        for row in db.select():
+            yield row.toatoms(add_additional_information=True)
+
+@task(
+    name="E-V Scan",
+    task_run_name=lambda: f"{task_run.task_name}: {task_run.parameters['atoms'].get_chemical_formula()} - {task_run.parameters['model'].name}",
+    cache_policy=TASK_SOURCE + INPUTS,
+)
+def ev_scan(atoms, model):
+    """
+    Perform an energy-volume scan for a given model and atomic structure.
+    
+    This function applies uniaxial strain to the structure in all three dimensions,
+    maintaining the fractional coordinates of atoms, and computes the energy at each
+    deformation point using the specified model.
+    
+    Args:
+        atoms: ASE atoms object containing the structure to analyze.
+        model: MLIPEnum model to use for the energy calculations.
+        
+    Returns:
+        dict: Results dictionary containing:
+            - method (str): The name of the model used
+            - id (str): The WBM ID of the structure
+            - eos (dict): Energy of state data with:
+                - volumes (list): Volume of the unit cell at each strain point
+                - energies (list): Computed potential energy at each strain point
+                
+    Note:
+        The strain range is fixed at ±20% with 21 evenly spaced points.
+        Results are also saved as a JSON file in a directory named after the model.
+    """
+    calculator = get_calculator(
+        model
+    )  # avoid sending entire model over prefect and select freer GPU
+
+    wbm_id = atoms.info["key_value_pairs"]["wbm_id"]
+
+    c0 = atoms.get_cell()
+    max_abs_strain = 0.2
+    npoints = 21
+    volumes = []
+    energies = []
+    for uniaxial_strain in np.linspace(-max_abs_strain, max_abs_strain, npoints):
+        cloned = atoms.copy()
+        scale_factor = uniaxial_strain + 1
+        cloned.set_cell(c0 * scale_factor, scale_atoms=True)
+        cloned.calc = calculator
+        volumes.append(cloned.get_volume())
+        energies.append(cloned.get_potential_energy())
+
+    data = {
+        "method": model.name,
+        "id": wbm_id,
+        "eos": {
+            "volumes": volumes, "energies": energies
+        }
+    }
+
+    fpath = Path(f"{model.name}") / f"{wbm_id}.json"
+    fpath.parent.mkdir(exist_ok=True)
+
+    df = pd.DataFrame([data])
+    df.to_json(fpath)
+
+    return df
+
+
+@flow
+def submit_tasks():
+    """
+    Create and submit energy-volume scan tasks for subsampled WBM structures and applicable models.
+    
+    This flow function:
+    1. Loads all structures from the WBM database
+    2. Iterates through available models in MLIPEnum
+    3. Filters models based on their capability to handle the 'wbm_ev' GPU task
+    4. Submits parallel ev_scan tasks for all valid (structure, model) combinations
+    5. Collects and returns results from all tasks
+    
+    Returns:
+        list: Results from all executed tasks (successful or failed)
+    """
+    futures = []
+    for atoms in load_wbm_structures():
+        for model in MLIPEnum:
+            if "wbm_ev" not in REGISTRY[model.name].get("gpu-tasks", []):
+                continue
+            try:
+                result = ev_scan.submit(atoms, model)
+            except Exception as e:
+                print(f"Failed to submit task for {model.name}: {e}")
+                continue
+            futures.append(result)
+    return [f.result(raise_on_failure=False) for f in futures]
+
+
+nodes_per_alloc = 1
+gpus_per_alloc = 1
+ntasks = 1
+
+cluster_kwargs = dict(
+    cores=1,
+    memory="64 GB",
+    processes=1,
+    shebang="#!/bin/bash",
+    account="m3828",
+    walltime="00:30:00",
+    # job_mem="0",
+    job_script_prologue=[
+        "source ~/.bashrc",
+        "module load python",
+        "source activate /pscratch/sd/c/cyrusyc/.conda/mlip-arena",
+    ],
+    job_directives_skip=["-n", "--cpus-per-task", "-J"],
+    job_extra_directives=[
+        "-J wbm_ev",
+        "-q debug",
+        f"-N {nodes_per_alloc}",
+        "-C gpu",
+        f"-G {gpus_per_alloc}",
+        "--exclusive",
+    ],
+)
+
+cluster = SLURMCluster(**cluster_kwargs)
+print(cluster.job_script())
+cluster.adapt(minimum_jobs=2, maximum_jobs=2)
+client = Client(cluster)
+
+submit_tasks.with_options(
+    task_runner=DaskTaskRunner(address=client.scheduler.address),
+    log_prints=True,
+)()

examples/wbm_ev/summary.csv

@@ -0,0 +1,10 @@
+model,rank,rank-aggregation,energy-diff-flip-times,tortuosity,spearman-compression-energy,spearman-compression-derivative,spearman-tension-energy,missing
+MACE-MPA,1,11,1.0,1.000675741122765,-0.9983393939393939,0.9993090909090908,0.9987181818181818,0
+CHGNet,2,14,1.0,1.0006287770651048,-0.9982787878787878,0.9439636363636364,0.999090909090909,0
+MatterSim,3,19,1.009,1.000567338639546,-0.9980969696969696,0.9997090909090908,0.9937541835359507,0
+eqV2(OMat),4,22,1.035,1.0008346292192054,-0.9982060606060604,0.9972242424242423,0.9986454545454545,0
+M3GNet,5,24,1.002,1.0020010929112253,-0.9975878787878787,0.997442424242424,0.9964676571137886,0
+ORBv2,6,29,1.058,1.004064906459821,-0.9977696969696969,0.970751515151515,0.9976,0
+SevenNet,7,33,1.034,1.0100246177550205,-0.9951636363636364,0.9465575757575757,0.9947048195608054,0
+MACE-MP(M),8,35,1.121,1.0807128149289842,-0.9438060606060605,0.9011878787878788,0.9987454545454546,0
+ALIGNN,9,46,3.909,1.3756517739089669,-0.8892069391323368,0.7602706775644651,0.862085379002138,0

examples/wbm_ev/summary.tex

@@ -0,0 +1,15 @@
+\begin{tabular}{lrrrrrrrl}
+\toprule
+model & rank & rank-aggregation & energy-diff-flip-times & tortuosity & spearman-compression-energy & spearman-compression-derivative & spearman-tension-energy & missing \\
+\midrule
+MACE-MPA & 1 & 11 & 1.000000 & 1.000676 & -0.998339 & 0.999309 & 0.998718 & 0 \\
+CHGNet & 2 & 14 & 1.000000 & 1.000629 & -0.998279 & 0.943964 & 0.999091 & 0 \\
+MatterSim & 3 & 19 & 1.009000 & 1.000567 & -0.998097 & 0.999709 & 0.993754 & 0 \\
+eqV2(OMat) & 4 & 22 & 1.035000 & 1.000835 & -0.998206 & 0.997224 & 0.998645 & 0 \\
+M3GNet & 5 & 24 & 1.002000 & 1.002001 & -0.997588 & 0.997442 & 0.996468 & 0 \\
+ORBv2 & 6 & 29 & 1.058000 & 1.004065 & -0.997770 & 0.970752 & 0.997600 & 0 \\
+SevenNet & 7 & 33 & 1.034000 & 1.010025 & -0.995164 & 0.946558 & 0.994705 & 0 \\
+MACE-MP(M) & 8 & 35 & 1.121000 & 1.080713 & -0.943806 & 0.901188 & 0.998745 & 0 \\
+ALIGNN & 9 & 46 & 3.909000 & 1.375652 & -0.889207 & 0.760271 & 0.862085 & 0 \\
+\bottomrule
+\end{tabular}

examples/wbm_structures.db

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc387c7787c21e7ff2ab80d5428c60b9e817c9b37f53e03c0f5e5e72dc44fe88
+size 782336

mlip_arena/models/externals/mattersim.py

@@ -5,9 +5,7 @@ from pathlib import Path
 import yaml
 from mattersim.forcefield import MatterSimCalculator
 
-from ase import Atoms
 from mlip_arena.models.utils import get_freer_device
-# from pymatgen.io.ase import AseAtomsAdaptor, MSONAtoms
 
 with open(Path(__file__).parents[1] / "registry.yaml", encoding="utf-8") as f:
     REGISTRY = yaml.safe_load(f)
@@ -24,6 +22,14 @@ class MatterSim(MatterSimCalculator):
             load_path=checkpoint, device=str(device or get_freer_device()), **kwargs
         )
 
+    def get_potential_energy(self, atoms=None, force_consistent=False):
+        return float(
+            super().get_potential_energy(
+                atoms=atoms,
+                force_consistent=force_consistent,
+            )
+        ) # mattersim return numpy float instead of python float
+
     def __getstate__(self):
         state = self.__dict__.copy()
 

mlip_arena/models/registry.yaml

@@ -15,6 +15,8 @@ MACE-MP(M):
     - homonuclear-diatomics
     - stability
     - combustion
+    - eos_bulk
+    - wbm_ev
   github: https://github.com/ACEsuit/mace
   doi: https://arxiv.org/abs/2401.00096
   date: 2023-12-29
@@ -38,6 +40,8 @@ CHGNet:
     - homonuclear-diatomics
     - stability
     - combustion
+    - eos_bulk
+    - wbm_ev
   github: https://github.com/CederGroupHub/chgnet
   doi: https://doi.org/10.1038/s42256-023-00716-3
   date: 2023-02-28
@@ -61,6 +65,8 @@ M3GNet:
     - homonuclear-diatomics
     - combustion
     - stability
+    - eos_bulk
+    - wbm_ev
   github: https://github.com/materialsvirtuallab/matgl
   doi: https://doi.org/10.1038/s43588-022-00349-3
   date: 2022-02-05
@@ -87,6 +93,8 @@ MatterSim:
     - homonuclear-diatomics
     - stability
     - combustion
+    - eos_bulk
+    - wbm_ev
   github: https://github.com/microsoft/mattersim
   doi: https://arxiv.org/abs/2405.04967
   date: 2024-12-05
@@ -113,6 +121,8 @@ ORBv2:
     - homonuclear-diatomics
     - combustion
     - stability
+    - eos_bulk
+    - wbm_ev
   github: https://github.com/orbital-materials/orb-models
   doi: https://arxiv.org/abs/2410.22570
   date: 2024-10-15
@@ -136,6 +146,8 @@ SevenNet:
     - homonuclear-diatomics
     - stability
     - combustion
+    - eos_bulk
+    - wbm_ev
   github: https://github.com/MDIL-SNU/SevenNet
   doi: https://doi.org/10.1021/acs.jctc.4c00190
   date: 2024-07-11
@@ -161,6 +173,7 @@ eqV2(OMat):
     - eos_alloy
   gpu-tasks:
     - homonuclear-diatomics
+    - wbm_ev
   prediction: EFS
   nvt: true
   npt: false # https://github.com/FAIR-Chem/fairchem/issues/888, https://github.com/atomind-ai/mlip-arena/issues/17
@@ -184,6 +197,8 @@ MACE-MPA:
   gpu-tasks:
     - homonuclear-diatomics
     - stability
+    - eos_bulk
+    - wbm_ev
   github: https://github.com/ACEsuit/mace
   doi: https://arxiv.org/abs/2401.00096
   date: 2024-12-09
@@ -336,6 +351,7 @@ ALIGNN:
   gpu-tasks:
     - homonuclear-diatomics
     - stability
+    - wbm_ev
     # - combustion
   prediction: EFS
   nvt: true

mlip_arena/tasks/optimize.py

@@ -78,7 +78,7 @@ def run(
 
     filter_kwargs = filter_kwargs or {}
     optimizer_kwargs = optimizer_kwargs or {}
-    criterion = criterion or {}
+    criterion = criterion or dict(steps=1000)
 
     if symmetry:
         atoms.set_constraint(FixSymmetry(atoms))

mlip_arena/tasks/registry.yaml

@@ -4,6 +4,18 @@ Homonuclear diatomics:
   task-layout: wide
   rank-page: homonuclear-diatomics
   last-update: 2024-09-19
+Energy-volume scans:
+  category: Fundamentals
+  task-page: wbm_ev
+  task-layout: wide
+  rank-page: wbm_ev
+  last-update: 2025-04-29
+Equation of state:
+  category: Fundamentals
+  task-page: eos_bulk
+  task-layout: wide
+  rank-page: eos_bulk
+  last-update: 2025-04-29
 Combustion:
   category: Molecular Dynamics
   task-page: combustion

serve/app.py

@@ -15,6 +15,8 @@ nav[""].append(leaderboard)
 wide_pages, centered_pages = [], []
 
 for task in TASKS:
+    if TASKS[task]['task-page'] is None:
+        continue
     page = st.Page(
         f"tasks/{TASKS[task]['task-page']}.py", title=task, icon=":material/target:"
     )
@@ -50,10 +52,10 @@ else:
     )
 
 
-st.toast(
-    "MLIP Arena is currently in **pre-alpha**. The results are not stable. Please interpret them with care. Contributions are welcome. For more information, visit https://github.com/atomind-ai/mlip-arena.",
-    icon="🍞",
-)
+# st.toast(
+#     "MLIP Arena is currently in **pre-alpha**. The results are not stable. Please interpret them with care. Contributions are welcome. For more information, visit https://github.com/atomind-ai/mlip-arena.",
+#     icon="🍞",
+# )
 
 st.sidebar.page_link(
     "https://github.com/atomind-ai/mlip-arena", label="GitHub Repository", icon=":material/code:"

serve/leaderboard.py

@@ -59,10 +59,10 @@ s = table.style.background_gradient(
     cmap="PuRd", subset=["Element Coverage"], vmin=0, vmax=120
 )
 
-st.warning(
-    "MLIP Arena is currently in **pre-alpha**. The results are not stable. Please interpret them with care.",
-    icon="⚠️",
-)
+# st.warning(
+#     "MLIP Arena is currently in **pre-alpha**. The results are not stable. Please interpret them with care.",
+#     icon="⚠️",
+# )
 st.info(
     "Contributions are welcome. For more information, visit https://github.com/atomind-ai/mlip-arena.",
     icon="🤗",
@@ -117,11 +117,12 @@ for task in TASKS:
 
     task_module = importlib.import_module(f"ranks.{TASKS[task]['rank-page']}")
 
-    st.page_link(
-        f"tasks/{TASKS[task]['task-page']}.py",
-        label="Go to the associated task page",
-        icon=":material/link:",
-    )
+    if TASKS[task]['task-page'] is not None:
+        st.page_link(
+            f"tasks/{TASKS[task]['task-page']}.py",
+            label="Go to the associated task page",
+            icon=":material/link:",
+        )
 
     #  Call the function from the imported module
     if hasattr(task_module, "render"):

serve/ranks/eos_bulk.py

@@ -0,0 +1,63 @@
+from pathlib import Path
+
+import pandas as pd
+import streamlit as st
+
+DATA_DIR = Path("examples/eos_bulk")
+
+
+table = pd.read_csv(DATA_DIR / "summary.csv")
+
+
+
+table = table.rename(
+    columns={
+        "model": "Model",
+        "rank": "Rank",
+        "rank-aggregation": "Rank aggr.",
+        "energy-diff-flip-times": "Derivative flips",
+        "tortuosity": "Tortuosity",
+        "spearman-compression-energy": "Spearman's coeff. (compression)",
+        "spearman-tension-energy": "Spearman's coeff. (tension)",
+        "spearman-compression-derivative": "Spearman's coeff. (compression derivative)",
+        "missing": "Missing",
+    },
+)
+
+table.set_index("Model", inplace=True)
+
+s = (
+    table.style.background_gradient(
+        cmap="Blues",
+        subset=["Rank", "Rank aggr."],
+    ).background_gradient(
+        cmap="Reds",
+        subset=[
+            "Spearman's coeff. (compression)",
+        ],
+    ).background_gradient(
+        cmap="Reds_r",
+        subset=[
+            "Spearman's coeff. (tension)",
+            "Spearman's coeff. (compression derivative)",
+        ],
+    ).background_gradient(
+        cmap="RdPu",
+        subset=["Tortuosity", "Derivative flips"],
+    ).format(
+        "{:.5f}",
+        subset=[
+            "Spearman's coeff. (compression)",
+            "Spearman's coeff. (tension)",
+            "Spearman's coeff. (compression derivative)",
+            "Tortuosity",
+            "Derivative flips",
+        ],
+    )
+)
+
+def render():
+    st.dataframe(
+        s,
+        use_container_width=True,
+    )

serve/ranks/wbm_ev.py

@@ -0,0 +1,63 @@
+from pathlib import Path
+
+import pandas as pd
+import streamlit as st
+
+DATA_DIR = Path("examples/wbm_ev")
+
+
+table = pd.read_csv(DATA_DIR / "summary.csv")
+
+
+
+table = table.rename(
+    columns={
+        "model": "Model",
+        "rank": "Rank",
+        "rank-aggregation": "Rank aggr.",
+        "energy-diff-flip-times": "Derivative flips",
+        "tortuosity": "Tortuosity",
+        "spearman-compression-energy": "Spearman's coeff. (compression)",
+        "spearman-tension-energy": "Spearman's coeff. (tension)",
+        "spearman-compression-derivative": "Spearman's coeff. (compression derivative)",
+        "missing": "Missing",
+    },
+)
+
+table.set_index("Model", inplace=True)
+
+s = (
+    table.style.background_gradient(
+        cmap="Blues",
+        subset=["Rank", "Rank aggr."],
+    ).background_gradient(
+        cmap="Reds",
+        subset=[
+            "Spearman's coeff. (compression)",
+        ],
+    ).background_gradient(
+        cmap="Reds_r",
+        subset=[
+            "Spearman's coeff. (tension)",
+            "Spearman's coeff. (compression derivative)",
+        ],
+    ).background_gradient(
+        cmap="RdPu",
+        subset=["Tortuosity", "Derivative flips"],
+    ).format(
+        "{:.5f}",
+        subset=[
+            "Spearman's coeff. (compression)",
+            "Spearman's coeff. (tension)",
+            "Spearman's coeff. (compression derivative)",
+            "Tortuosity",
+            "Derivative flips",
+        ],
+    )
+)
+
+def render():
+    st.dataframe(
+        s,
+        use_container_width=True,
+    )