Integrate HIF with more of XGI (#613)

* move hif functionality to convert

* format with isort and black

* add docs

* add collection handling

* add warning

* add docstrings

* added HIF to `load_xgi_data`

* added more close to tests

* Update xgi_data.py

* Update xgi_data.py

* remove other changes

* added unit tests

* Update xgi_data.py

* Update HOW_TO_CONTRIBUTE.md

* response to review

* Response to review

* Update docs.txt

* Update release.txt

HOW_TO_CONTRIBUTE.md

@@ -9,7 +9,7 @@ Please note we have a [code of conduct](/CODE_OF_CONDUCT.md), please follow it i
 1. Download the dependencies in the developer [requirements file](/requirements/developer.txt).
 2. Add unit tests for features being added or bugs being fixed.
 3. Include any new method/function in the corresponding docs file.
-4. Run `pytest` to verify all unit tests pass.
+4. Run `pytest` to verify all unit tests pass. (To see what lines are covered, read the [`pytest-cov`](https://pytest-cov.readthedocs.io/en/latest/reporting.html) documentation.)
 5. [OPTIONAL] Format codebase according to the steps below.
 5. Submit Pull Request with a list of changes, links to issues that it addresses (if applicable)
 6. You may merge the Pull Request in once you have the sign-off of at least one other developer, or if you do not have permission to do that, you may request the reviewer to merge it for you.

docs/source/api/convert.rst

@@ -11,6 +11,7 @@ convert package
    ~xgi.convert.bipartite_graph
    ~xgi.convert.encapsulation_dag
    ~xgi.convert.graph
+   ~xgi.convert.hif_dict
    ~xgi.convert.higher_order_network
    ~xgi.convert.hyperedges
    ~xgi.convert.hypergraph_dict

docs/source/api/convert/xgi.convert.hif_dict.rst

@@ -0,0 +1,11 @@
+xgi.convert.hif_dict
+====================
+
+.. currentmodule:: xgi.convert.hif_dict
+
+.. automodule:: xgi.convert.hif_dict
+
+   .. rubric:: Functions
+
+   .. autofunction:: to_hif_dict
+   .. autofunction:: from_hif_dict

docs/source/api/readwrite/xgi.readwrite.hif.rst

@@ -8,4 +8,6 @@ xgi.readwrite.hif
    .. rubric:: Functions
 
    .. autofunction:: read_hif
+   .. autofunction:: read_hif_collection
    .. autofunction:: write_hif
+   .. autofunction:: write_hif_collection

docs/source/xgi-data.rst

@@ -30,7 +30,9 @@ See the `load_bigg_data() documentation <api/readwrite/xgi.readwrite.bigg_data.h
 Dataset format
 --------------
 
-The xgi-data format for higher-order datasets is a JSON data structure with the following structure:
+XGI-DATA provides higher-order datasets in two formats: (1) Hypergraph Interchange Format (HIF)-compliant JSON (2) XGI-specific JSON format. See the `HIF-standard <https://github.com/pszufe/HIF-standard>`_ documentation on format (1).
+All future datasets and updates to current datasets will be stored as format (1).
+Format (2) is structured as follows:
 
 * :code:`hypergraph-data`: This tag accesses the attributes of the entire hypergraph dataset such as the authors or dataset name.
 

requirements/release.txt

@@ -3,4 +3,3 @@
 twine>=3.4
 build>=1.2.1
 wheel>=0.36
-github-changelog

tests/convert/test_hif_dict.py

@@ -0,0 +1,226 @@
+import pytest
+
+import xgi
+
+
+def test_to_hif_dict(
+    edgelist1,
+    hyperwithdupsandattrs,
+    simplicialcomplex1,
+    diedgedict1,
+    dihyperwithattrs,
+):
+    H = xgi.Hypergraph(edgelist1)
+    d = xgi.to_hif_dict(H)
+
+    assert "nodes" not in d
+    assert "edges" not in d
+    assert "incidences" in d
+    assert "network-type" in d
+    assert d["network-type"] == "undirected"
+
+    incidences = [
+        {"edge": 0, "node": 1},
+        {"edge": 0, "node": 2},
+        {"edge": 0, "node": 3},
+        {"edge": 1, "node": 4},
+        {"edge": 2, "node": 5},
+        {"edge": 2, "node": 6},
+        {"edge": 3, "node": 6},
+        {"edge": 3, "node": 7},
+        {"edge": 3, "node": 8},
+    ]
+    assert sorted(d["incidences"], key=lambda x: (x["edge"], x["node"])) == incidences
+
+    # hypergraph with attributes
+    hyperwithdupsandattrs["name"] = "test"
+    d = xgi.to_hif_dict(hyperwithdupsandattrs)
+
+    assert "nodes" in d
+    assert "edges" in d
+    assert "incidences" in d
+    assert "network-type" in d
+    assert d["network-type"] == "undirected"
+    assert "metadata" in d
+    assert d["metadata"] == {"name": "test"}
+
+    nodes = [
+        {"node": 1, "attrs": {"color": "red", "name": "horse"}},
+        {"node": 2, "attrs": {"color": "blue", "name": "pony"}},
+        {"node": 3, "attrs": {"color": "yellow", "name": "zebra"}},
+        {"node": 4, "attrs": {"color": "red", "name": "orangutan", "age": 20}},
+        {"node": 5, "attrs": {"color": "blue", "name": "fish", "age": 2}},
+    ]
+
+    edges = [
+        {"edge": 0, "attrs": {"color": "blue"}},
+        {"edge": 1, "attrs": {"color": "red", "weight": 2}},
+        {"edge": 2, "attrs": {"color": "yellow"}},
+        {"edge": 3, "attrs": {"color": "purple"}},
+        {"edge": 4, "attrs": {"color": "purple", "name": "test"}},
+    ]
+
+    incidences = [
+        {"edge": 0, "node": 1},
+        {"edge": 0, "node": 2},
+        {"edge": 1, "node": 1},
+        {"edge": 1, "node": 2},
+        {"edge": 2, "node": 1},
+        {"edge": 2, "node": 2},
+        {"edge": 3, "node": 3},
+        {"edge": 3, "node": 4},
+        {"edge": 3, "node": 5},
+        {"edge": 4, "node": 3},
+        {"edge": 4, "node": 4},
+        {"edge": 4, "node": 5},
+    ]
+
+    assert sorted(d["nodes"], key=lambda x: x["node"]) == nodes
+    assert d["edges"] == edges
+    assert sorted(d["incidences"], key=lambda x: (x["edge"], x["node"])) == incidences
+
+    # Simplicial complexes
+    d = xgi.to_hif_dict(simplicialcomplex1)
+
+    assert "nodes" not in d
+    assert "edges" not in d
+    assert "incidences" in d
+    assert "network-type" in d
+    assert d["network-type"] == "asc"
+
+    incidences = [
+        {"edge": "e1", "node": 0},
+        {"edge": "e1", "node": "b"},
+        {"edge": "e2", "node": 0},
+        {"edge": "e2", "node": "c"},
+        {"edge": "e3", "node": 0},
+        {"edge": "e3", "node": "b"},
+        {"edge": "e3", "node": "c"},
+        {"edge": "e4", "node": "b"},
+        {"edge": "e4", "node": "c"},
+    ]
+
+    def _mixed(ele):
+        return (0, int(ele)) if isinstance(ele, int) else (1, ele)
+
+    sorted_incidences = sorted(
+        d["incidences"], key=lambda x: (_mixed(x["edge"]), _mixed(x["node"]))
+    )
+    assert sorted_incidences == incidences
+
+    # dihypergraphs without attributes
+    H = xgi.DiHypergraph(diedgedict1)
+
+    d = xgi.to_hif_dict(H)
+
+    assert "nodes" not in d
+    assert "edges" not in d
+    assert "incidences" in d
+
+    # dihypergraphs with attributes
+    d = xgi.to_hif_dict(dihyperwithattrs)
+
+    assert "nodes" in d
+    assert "edges" in d
+    assert "incidences" in d
+
+
+def test_from_hif_dict(
+    hyperwithdupsandattrs,
+    simplicialcomplex1,
+    dihyperwithattrs,
+):
+    d = xgi.to_hif_dict(hyperwithdupsandattrs)
+
+    # test basic import
+    H = xgi.from_hif_dict(d)
+
+    assert isinstance(H, xgi.Hypergraph)
+    assert (H.num_nodes, H.num_edges) == (5, 5)
+    assert set(H.nodes) == {1, 2, 3, 4, 5}
+    assert H.nodes[1] == {"color": "red", "name": "horse"}
+    assert H.nodes[2] == {"color": "blue", "name": "pony"}
+    assert H.nodes[3] == {"color": "yellow", "name": "zebra"}
+    assert H.nodes[4] == {"color": "red", "name": "orangutan", "age": 20}
+    assert H.nodes[5] == {"color": "blue", "name": "fish", "age": 2}
+
+    assert set(H.edges) == {0, 1, 2, 3, 4}
+    assert H.edges[0] == {"color": "blue"}
+    assert H.edges[1] == {"color": "red", "weight": 2}
+    assert H.edges[2] == {"color": "yellow"}
+    assert H.edges[3] == {"color": "purple"}
+    assert H.edges[4] == {"color": "purple", "name": "test"}
+
+    edgedict = {0: {1, 2}, 1: {1, 2}, 2: {1, 2}, 3: {3, 4, 5}, 4: {3, 4, 5}}
+    assert H.edges.members(dtype=dict) == edgedict
+
+    # cast nodes and edges
+    H = xgi.from_hif_dict(d, nodetype=str, edgetype=float)
+    assert set(H.nodes) == {"1", "2", "3", "4", "5"}
+    assert set(H.edges) == {0.0, 1.0, 2.0, 3.0, 4.0}
+
+    assert H.nodes["1"] == {"color": "red", "name": "horse"}
+    assert H.edges[0.0] == {"color": "blue"}
+
+    edgedict = {
+        0: {"1", "2"},
+        1: {"1", "2"},
+        2: {"1", "2"},
+        3: {"3", "4", "5"},
+        4: {"3", "4", "5"},
+    }
+    assert H.edges.members(dtype=dict) == edgedict
+
+    ds = xgi.to_hif_dict(simplicialcomplex1)
+
+    S = xgi.from_hif_dict(ds)
+    assert isinstance(S, xgi.SimplicialComplex)
+    assert (S.num_nodes, S.num_edges) == (3, 4)
+
+    assert set(S.nodes) == {0, "b", "c"}
+    assert set(S.edges) == {"e1", "e2", "e3", "e4"}
+
+    # dihypergraphs
+    d = xgi.to_hif_dict(dihyperwithattrs)
+
+    DH = xgi.from_hif_dict(d)
+    assert (DH.num_nodes, DH.num_edges) == (6, 3)
+    assert isinstance(DH, xgi.DiHypergraph)
+    assert set(DH.nodes) == {0, 1, 2, 3, 4, 5}
+    assert set(DH.edges) == {0, 1, 2}
+
+    edgedict = {0: ({0, 1}, {2}), 1: ({1, 2}, {4}), 2: ({2, 3, 4}, {4, 5})}
+    assert DH.edges.dimembers(dtype=dict) == edgedict
+
+    # test error checking
+    with pytest.raises(TypeError):
+        S = xgi.from_hif_dict(ds, edgetype=int)
+
+    # metadata
+    hyperwithdupsandattrs["name"] = "test"
+    d = xgi.to_hif_dict(hyperwithdupsandattrs)
+    H = xgi.from_hif_dict(d)
+
+    assert H["name"] == "test"
+
+    # test isolates and empty edges
+    H = xgi.Hypergraph()
+    H.add_nodes_from(range(5))
+    H.add_edges_from([[1, 2, 3], []])
+
+    d = xgi.to_hif_dict(H)
+
+    H = xgi.from_hif_dict(d)
+    assert H.edges.size.aslist() == [3, 0]
+    assert set(H.nodes.isolates()) == {0, 4}
+
+    H = xgi.DiHypergraph()
+    H.add_nodes_from(range(5))
+    H.add_edges_from([([1, 2, 3], [2, 4]), [[], []]])
+
+    d = xgi.to_hif_dict(H)
+
+    H = xgi.from_hif_dict(d)
+
+    assert H.edges.size.aslist() == [4, 0]
+    assert set(H.nodes.isolates()) == {0}