Skip to content

Commit

Permalink
Refactor popgetter init
Browse files Browse the repository at this point in the history
When adding a new country, it just needs to be added to
popgetter/assets/__init__.py once.

Closes #111
  • Loading branch information
penelopeysm committed Jun 8, 2024
1 parent f5d8aeb commit b517c8c
Show file tree
Hide file tree
Showing 2 changed files with 16 additions and 25 deletions.
37 changes: 12 additions & 25 deletions python/popgetter/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,13 +48,15 @@
UnresolvedAssetJobDefinition,
)

from popgetter import assets, azure_test, cloud_outputs
from popgetter import azure_test, cloud_outputs
from popgetter.assets import countries


all_assets: Sequence[AssetsDefinition | SourceAsset | CacheableAssetsDefinition] = [
*load_assets_from_package_module(assets.us, group_name="us"),
*load_assets_from_package_module(assets.be, group_name="be"),
*load_assets_from_package_module(assets.uk, group_name="uk"),
*load_assets_from_package_module(assets.ni, group_name="ni"),
*[asset
for (module, name) in countries
for asset in load_assets_from_package_module(module, group_name=name)
],
*load_assets_from_package_module(cloud_outputs, group_name="cloud_outputs"),
*(
load_assets_from_modules([azure_test], group_name="azure_test")
Expand All @@ -65,26 +67,11 @@

jobs: list[UnresolvedAssetJobDefinition] = [
define_asset_job(
name="job_be",
selection=AssetSelection.groups("be"),
description="Downloads Belgian data.",
partitions_def=assets.be.census_tables.dataset_node_partition,
),
define_asset_job(
name="job_us",
selection=AssetSelection.groups("us"),
description="Downloads USA data.",
),
define_asset_job(
name="job_uk",
selection=AssetSelection.groups("uk"),
description="Downloads UK data.",
),
define_asset_job(
name="job_ni",
selection=AssetSelection.groups("ni"),
description="Downloads Northern Ireland data.",
),
name=f"job_{country_name}",
selection=AssetSelection.groups(country_name),
description=f"Downloads data for country '{country_name}'.",
)
for (_, country_name) in countries
]


Expand Down
4 changes: 4 additions & 0 deletions python/popgetter/assets/__init__.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
from __future__ import annotations

from . import be, ni, uk, us # noqa: F401

countries = [(mod, mod.__name__.split(".")[-1]) for mod in [be, ni, uk, us]]

__all__ = ["countries"]

0 comments on commit b517c8c

Please sign in to comment.