Skip to content

Commit

Permalink
Produce SDMX structural metadata for transport
Browse files Browse the repository at this point in the history
- Add ExogenousDataFile.generate_dfd().
- Add tests.
  • Loading branch information
khaeru committed Aug 12, 2024
1 parent 9aa5928 commit e9aca5d
Show file tree
Hide file tree
Showing 2 changed files with 53 additions and 0 deletions.
48 changes: 48 additions & 0 deletions message_ix_models/model/transport/files.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
if TYPE_CHECKING:
import genno
from genno.core.key import KeyLike
from sdmx.model.common import BaseDataflowDefinition

from message_ix_models import Context

Expand Down Expand Up @@ -86,6 +87,53 @@ def add_tasks(
c.add("load_file", path, key=self.key, dims=dims, name=self.key.name)
return (self.key,)

def generate_dfd(self) -> "BaseDataflowDefinition":
from importlib.metadata import version

from ixmp.report.util import get_reversed_rename_dims
from packaging.version import parse
from sdmx.model.common import ConceptScheme
from sdmx.model.v21 import DataflowDefinition, DataStructureDefinition

from message_ix_models.util.sdmx import read

# Read the existing agency scheme
ece = read("IIASA_ECE:AGENCIES")["IIASA_ECE"]
name_for_id = self.key.name.upper().replace(" ", "_")
version = parse(version("message_ix_models")).base_version
ma_kwargs = dict(maintainer=ece, version=version)

# Create a shared concept scheme
cs = ConceptScheme(id="CS_MESSAGE_TRANSPORT", **ma_kwargs)

# Create a data structure definition
dsd = DataStructureDefinition(
id=f"DS_{name_for_id}", **ma_kwargs, name=self.doc
)

# Add dimensions
dims = get_reversed_rename_dims()
for dim in self.key.dims:
# Symbol ('n') → Dimension ID ('node') → upper case
dim_id = dims.get(dim, dim).upper()

concept = cs.setdefault(id="dim_id")
dsd.dimensions.getdefault(id=dim_id, concept_identity=concept)

dfd = DataflowDefinition(
id=f"DF_{name_for_id}", **ma_kwargs, name=self.doc, structure=dsd
)
# TODO Add annotations: preferred file name

# TODO Generate a CSV template file
# 1. In the current format.abs
# 2. In SDMX-CSV.
# dm = DataMessage()
# dm.data.append(DataSet(structure))
# template =

return dfd


ExogenousDataFile(
"pdt-cap-ref",
Expand Down
5 changes: 5 additions & 0 deletions message_ix_models/tests/model/transport/test_files.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,3 +43,8 @@ def test_configure_build(

# Dimensions are as expected
assert set(Key(result).dims) == set(file.key.dims)

@pytest.mark.parametrize("file", FILES, ids=lambda f: "-".join(f.parts))
def test_generate_dfd(self, file) -> None:
dfd = file.generate_dfd()
del dfd

0 comments on commit e9aca5d

Please sign in to comment.