diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 5952f8959..2613f7d6a 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -23,6 +23,12 @@ else() ) endif() +install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/podio_gen + DESTINATION ${podio_PYTHON_INSTALLDIR} + REGEX test_.*\\.py$ EXCLUDE # Do not install test files + PATTERN __pycache__ EXCLUDE # Or pythons caches + ) + #--- install templates --------------------------------------------------------- install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/templates DESTINATION ${podio_PYTHON_INSTALLDIR}) @@ -30,10 +36,12 @@ install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/templates IF (BUILD_TESTING) add_test( NAME pyunittest COMMAND python3 -m unittest discover -s ${PROJECT_SOURCE_DIR}/python/podio) PODIO_SET_TEST_ENV(pyunittest) + add_test( NAME pyunittest_gen COMMAND python3 -m unittest discover -s ${PROJECT_SOURCE_DIR}/python/podio_gen) + PODIO_SET_TEST_ENV(pyunittest_gen) set_property(TEST pyunittest PROPERTY DEPENDS write write_frame_root) if (TARGET write_sio) set_property(TEST pyunittest PROPERTY DEPENDS write_sio write_frame_sio) endif() - set_property(TEST pyunittest PROPERTY WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/tests) + set_tests_properties(pyunittest pyunittest_gen PROPERTIES WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/tests) ENDIF() diff --git a/python/podio/__init__.py b/python/podio/__init__.py index 1414f58be..fd5fa7209 100644 --- a/python/podio/__init__.py +++ b/python/podio/__init__.py @@ -1,56 +1,33 @@ """Python module for the podio EDM toolkit and its utilities""" -import sys - from .__version__ import __version__ -from .podio_config_reader import * # noqa: F403, F401 - -import ROOT # pylint: disable=wrong-import-order - -# Track whether we were able to dynamially load the library that is built by -# podio and enable certain features of the bindings only if they are actually -# available. -_DYNAMIC_LIBS_LOADED = False - -# Check if we can locate the dictionary wthout loading it as this allows us to -# silence any ouptput. If we can find it, we can also safely load it -if ROOT.gSystem.DynamicPathName("libpodioDict.so", True): - ROOT.gSystem.Load("libpodioDict.so") - from ROOT import podio - - _DYNAMIC_LIBS_LOADED = True - -if _DYNAMIC_LIBS_LOADED: - from .frame import Frame - from . import root_io, reading - - try: - # We try to import the sio bindings which may fail if ROOT is not able to - # load the dictionary in this case they have most likely not been built and - # we just move on - from . import sio_io - except ImportError: - pass - - from . import EventStore - - try: - # For some reason the test_utils only work at (test) runtime if they are - # imported with the rest of podio. Otherwise they produce a weird c++ error. - # This happens even if we import the *exact* same file. - from . import test_utils # noqa: F401 - except ImportError: - pass - - # Make sure that this module is actually usable as podio even though most of - # it is dynamically populated by cppyy - sys.modules["podio"] = podio - - __all__ = [ - "__version__", - "Frame", - "root_io", - "sio_io", - "reading", - "EventStore" - ] +# Try to load podio, this is equivalent to trying to load libpodio.so and will +# error if libpodio.so is not found but work if it's found +try: + from ROOT import podio # noqa: F401 +except ImportError: + print('Unable to load podio, make sure that libpodio.so is in LD_LIBRARY_PATH') + raise + +from .frame import Frame +from . import root_io, reading + +try: + # We try to import the sio bindings which may fail if ROOT is not able to + # load the dictionary. In this case they have most likely not been built and + # we just move on + from . import sio_io +except ImportError: + pass + +from . import EventStore + + +__all__ = [ + "__version__", + "Frame", + "root_io", + "sio_io", + "reading", + "EventStore" + ] diff --git a/python/podio/sio_io.py b/python/podio/sio_io.py index 30257a860..a0d45236b 100644 --- a/python/podio/sio_io.py +++ b/python/podio/sio_io.py @@ -2,9 +2,9 @@ """Python module for reading sio files containing podio Frames""" from ROOT import gSystem -ret = gSystem.Load('libpodioSioIO') # noqa: 402 -# Return values: -1 when it doesn't exist and -2 when there is a version mismatch -if ret < 0: +if gSystem.DynamicPathName("libpodioSioIO.so", True): + gSystem.Load('libpodioSioIO') # noqa: 402 +else: raise ImportError('Error when importing libpodioSioIO') from ROOT import podio # noqa: 402 # pylint: disable=wrong-import-position diff --git a/python/podio/test_Frame.py b/python/podio/test_Frame.py index 0dc823d76..1ed334a7b 100644 --- a/python/podio/test_Frame.py +++ b/python/podio/test_Frame.py @@ -4,7 +4,7 @@ import unittest # pylint: disable=import-error -from test_utils import ExampleHitCollection # noqa: E402 +from ROOT import ExampleHitCollection from podio.frame import Frame # using root_io as that should always be present regardless of which backends are built diff --git a/python/podio/test_utils.py b/python/podio/test_utils.py index 2c07d086b..948571293 100644 --- a/python/podio/test_utils.py +++ b/python/podio/test_utils.py @@ -2,59 +2,5 @@ """Utilities for python unittests""" import os -import ROOT - -if ROOT.gSystem.DynamicPathName("libTestDataModelDict.so", True): # noqa: E402 - ROOT.gSystem.Load("libTestDataModelDict.so") # noqa: E402 -from ROOT import ExampleHitCollection, ExampleClusterCollection # noqa: E402 # pylint: disable=wrong-import-position - -from podio import Frame # pylint: disable=wrong-import-position - SKIP_SIO_TESTS = os.environ.get("SKIP_SIO_TESTS", "1") == "1" - - -def create_hit_collection(): - """Create a simple hit collection with two hits for testing""" - hits = ExampleHitCollection() - hits.create(0xBAD, 0.0, 0.0, 0.0, 23.0) - hits.create(0xCAFFEE, 1.0, 0.0, 0.0, 12.0) - - return hits - - -def create_cluster_collection(): - """Create a simple cluster collection with two clusters""" - clusters = ExampleClusterCollection() - clu0 = clusters.create() - clu0.energy(3.14) - clu1 = clusters.create() - clu1.energy(1.23) - - return clusters - - -def create_frame(): - """Create a frame with an ExampleHit and an ExampleCluster collection""" - frame = Frame() - hits = create_hit_collection() - frame.put(hits, "hits_from_python") - clusters = create_cluster_collection() - frame.put(clusters, "clusters_from_python") - - frame.put_parameter("an_int", 42) - frame.put_parameter("some_floats", [1.23, 7.89, 3.14]) - frame.put_parameter("greetings", ["from", "python"]) - frame.put_parameter("real_float", 3.14, as_type="float") - frame.put_parameter("more_real_floats", [1.23, 4.56, 7.89], as_type="float") - - return frame - - -def write_file(writer_type, filename): - """Write a file using the given Writer type and put one Frame into it under - the events category - """ - writer = writer_type(filename) - event = create_frame() - writer.write_frame(event, "events") diff --git a/python/podio_class_generator.py b/python/podio_class_generator.py index ad4cd5185..14d96f606 100755 --- a/python/podio_class_generator.py +++ b/python/podio_class_generator.py @@ -16,8 +16,8 @@ from podio_schema_evolution import DataModelComparator # dealing with cyclic imports from podio_schema_evolution import RenamedMember, root_filter, RootIoRule -from podio.podio_config_reader import PodioConfigReader -from podio.generator_utils import DataType, DefinitionError, DataModelJSONEncoder +from podio_gen.podio_config_reader import PodioConfigReader +from podio_gen.generator_utils import DataType, DefinitionError, DataModelJSONEncoder THIS_DIR = os.path.dirname(os.path.abspath(__file__)) TEMPLATE_DIR = os.path.join(THIS_DIR, 'templates') diff --git a/python/podio_gen/__init__.py b/python/podio_gen/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/podio/generator_utils.py b/python/podio_gen/generator_utils.py similarity index 100% rename from python/podio/generator_utils.py rename to python/podio_gen/generator_utils.py diff --git a/python/podio/podio_config_reader.py b/python/podio_gen/podio_config_reader.py similarity index 99% rename from python/podio/podio_config_reader.py rename to python/podio_gen/podio_config_reader.py index 303632367..b7e3790b0 100644 --- a/python/podio/podio_config_reader.py +++ b/python/podio_gen/podio_config_reader.py @@ -6,7 +6,7 @@ import warnings import yaml -from podio.generator_utils import MemberVariable, DefinitionError, BUILTIN_TYPES, DataModel +from podio_gen.generator_utils import MemberVariable, DefinitionError, BUILTIN_TYPES, DataModel class MemberParser: diff --git a/python/podio/test_ClassDefinitionValidator.py b/python/podio_gen/test_ClassDefinitionValidator.py similarity index 99% rename from python/podio/test_ClassDefinitionValidator.py rename to python/podio_gen/test_ClassDefinitionValidator.py index c1b9d4cd1..42ad5903a 100644 --- a/python/podio/test_ClassDefinitionValidator.py +++ b/python/podio_gen/test_ClassDefinitionValidator.py @@ -8,8 +8,8 @@ import unittest from copy import deepcopy -from podio.podio_config_reader import ClassDefinitionValidator, MemberVariable, DefinitionError -from podio.generator_utils import DataModel +from podio_gen.podio_config_reader import ClassDefinitionValidator, MemberVariable, DefinitionError +from podio_gen.generator_utils import DataModel def make_dm(components, datatypes, options=None): diff --git a/python/podio/test_DataModelJSONEncoder.py b/python/podio_gen/test_DataModelJSONEncoder.py similarity index 96% rename from python/podio/test_DataModelJSONEncoder.py rename to python/podio_gen/test_DataModelJSONEncoder.py index b63ff22f0..b69055681 100644 --- a/python/podio/test_DataModelJSONEncoder.py +++ b/python/podio_gen/test_DataModelJSONEncoder.py @@ -3,8 +3,8 @@ import unittest -from podio.generator_utils import DataModelJSONEncoder -from podio.podio_config_reader import MemberParser +from podio_gen.generator_utils import DataModelJSONEncoder +from podio_gen.podio_config_reader import MemberParser def get_member_var_json(string): diff --git a/python/podio/test_MemberParser.py b/python/podio_gen/test_MemberParser.py similarity index 99% rename from python/podio/test_MemberParser.py rename to python/podio_gen/test_MemberParser.py index d99cd11d2..b860dc833 100644 --- a/python/podio/test_MemberParser.py +++ b/python/podio_gen/test_MemberParser.py @@ -6,7 +6,7 @@ import unittest -from podio.podio_config_reader import MemberParser, DefinitionError +from podio_gen.podio_config_reader import MemberParser, DefinitionError class MemberParserTest(unittest.TestCase): diff --git a/python/podio_schema_evolution.py b/python/podio_schema_evolution.py index e2b7a916f..b2c3b082b 100755 --- a/python/podio_schema_evolution.py +++ b/python/podio_schema_evolution.py @@ -5,7 +5,7 @@ import yaml -from podio.podio_config_reader import PodioConfigReader +from podio_gen.podio_config_reader import PodioConfigReader # @TODO: not really a good class model here diff --git a/tests/root_io/CMakeLists.txt b/tests/root_io/CMakeLists.txt index e719e583f..17435bedc 100644 --- a/tests/root_io/CMakeLists.txt +++ b/tests/root_io/CMakeLists.txt @@ -80,6 +80,6 @@ endforeach() #--- Write via python and the ROOT backend and see if we can read it back in in #--- c++ -add_test(NAME write_python_frame_root COMMAND python3 ${CMAKE_CURRENT_LIST_DIR}/write_frame_root.py) +add_test(NAME write_python_frame_root COMMAND python3 ${PROJECT_SOURCE_DIR}/tests/write_frame.py example_frame_with_py.root) PODIO_SET_TEST_ENV(write_python_frame_root) set_property(TEST read_python_frame_root PROPERTY DEPENDS write_python_frame_root) diff --git a/tests/root_io/write_frame_root.py b/tests/root_io/write_frame_root.py deleted file mode 100644 index e0525a268..000000000 --- a/tests/root_io/write_frame_root.py +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env python3 -"""Script to write a Frame in ROOT format""" - -from podio import test_utils - -from podio.root_io import Writer # pylint: disable=wrong-import-position - - -test_utils.write_file(Writer, "example_frame_with_py.root") diff --git a/tests/sio_io/CMakeLists.txt b/tests/sio_io/CMakeLists.txt index c5a15b34f..d48ad1372 100644 --- a/tests/sio_io/CMakeLists.txt +++ b/tests/sio_io/CMakeLists.txt @@ -8,6 +8,7 @@ set(sio_dependent_tests write_frame_sio.cpp read_frame_legacy_sio.cpp read_and_write_frame_sio.cpp + read_python_frame_sio.cpp ) set(sio_libs podio::podioSioIO) foreach( sourcefile ${sio_dependent_tests} ) @@ -36,3 +37,9 @@ set_tests_properties( add_test(NAME check_benchmark_outputs_sio COMMAND check_benchmark_outputs write_benchmark_sio.root read_benchmark_sio.root) set_property(TEST check_benchmark_outputs_sio PROPERTY DEPENDS read_timed_sio write_timed_sio) + +#--- Write via python and the SIO backend and see if we can read it back in in +#--- c++ +add_test(NAME write_python_frame_sio COMMAND python3 ${PROJECT_SOURCE_DIR}/tests/write_frame.py example_frame_with_py.sio) +PODIO_SET_TEST_ENV(write_python_frame_sio) +set_property(TEST read_python_frame_sio PROPERTY DEPENDS write_python_frame_sio) diff --git a/tests/sio_io/write_frame_sio.py b/tests/sio_io/write_frame_sio.py deleted file mode 100644 index 94e08aa27..000000000 --- a/tests/sio_io/write_frame_sio.py +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env python3 -"""Script to write a Frame in SIO format""" - -from podio import test_utils -from podio.sio_io import Writer - -test_utils.write_file(Writer, "example_frame_with_py.sio") diff --git a/tests/write_frame.py b/tests/write_frame.py new file mode 100644 index 000000000..05b901f5e --- /dev/null +++ b/tests/write_frame.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""Utilities for python unittests""" + +import importlib + + +import ROOT + +if ROOT.gSystem.Load("libTestDataModelDict.so") < 0: # noqa: E402 + raise RuntimeError("Could not load TestDataModel dictionary") + +from ROOT import ( # pylint: disable=wrong-import-position + ExampleHitCollection, + ExampleClusterCollection, + ) # noqa: E402 + +from podio import Frame # pylint: disable=wrong-import-position + + +def create_hit_collection(): + """Create a simple hit collection with two hits for testing""" + hits = ExampleHitCollection() + hits.create(0xBAD, 0.0, 0.0, 0.0, 23.0) + hits.create(0xCAFFEE, 1.0, 0.0, 0.0, 12.0) + + return hits + + +def create_cluster_collection(): + """Create a simple cluster collection with two clusters""" + clusters = ExampleClusterCollection() + clu0 = clusters.create() + clu0.energy(3.14) + clu1 = clusters.create() + clu1.energy(1.23) + + return clusters + + +def create_frame(): + """Create a frame with an ExampleHit and an ExampleCluster collection""" + frame = Frame() + hits = create_hit_collection() + frame.put(hits, "hits_from_python") + clusters = create_cluster_collection() + frame.put(clusters, "clusters_from_python") + + frame.put_parameter("an_int", 42) + frame.put_parameter("some_floats", [1.23, 7.89, 3.14]) + frame.put_parameter("greetings", ["from", "python"]) + frame.put_parameter("real_float", 3.14, as_type="float") + frame.put_parameter("more_real_floats", [1.23, 4.56, 7.89], as_type="float") + + return frame + + +def write_file(io_backend, filename): + """Write a file using the given Writer type and put one Frame into it under + the events category + """ + io_module = importlib.import_module(f"podio.{io_backend}") + + writer = io_module.Writer(filename) + event = create_frame() + writer.write_frame(event, "events") + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument("outputfile", help="Output file name") + + args = parser.parse_args() + + io_format = args.outputfile.split(".")[-1] + + write_file(f"{io_format}_io", args.outputfile) diff --git a/tools/podio-vis b/tools/podio-vis index 032feb96e..667b5feab 100755 --- a/tools/podio-vis +++ b/tools/podio-vis @@ -4,7 +4,7 @@ import sys import argparse import yaml -from podio.podio_config_reader import PodioConfigReader +from podio_gen.podio_config_reader import PodioConfigReader try: from graphviz import Digraph except ImportError: