Skip to content

Commit

Permalink
Merge branch 'serialisation-aliases' into harmonise-data-metrics
Browse files Browse the repository at this point in the history
  • Loading branch information
@penelopeysm
penelopeysm committed Jul 2, 2024
2 parents 2ac2c37 + 38c1793 commit 4593779
Show file tree
Hide file tree
Showing 4 changed files with 48 additions and 37 deletions.
15 changes: 9 additions & 6 deletions src/display.rs
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
use comfy_table::{presets::NOTHING, *};
use itertools::izip;
use popgetter::search::SearchResults;
use popgetter::{search::SearchResults, COL};

pub fn display_search_results(results: SearchResults, max_results: Option<usize>) {
let df_to_show = match max_results {
Expand All @@ -9,11 +9,14 @@ pub fn display_search_results(results: SearchResults, max_results: Option<usize>
};

for (metric_id, hrn, desc, hxl, level) in izip!(
df_to_show.column("metric_id").unwrap().iter(),
df_to_show.column("human_readable_name").unwrap().iter(),
df_to_show.column("metric_description").unwrap().iter(),
df_to_show.column("metric_hxl_tag").unwrap().iter(),
df_to_show.column("geometry_level").unwrap().iter(),
df_to_show.column(COL::METRIC_ID).unwrap().iter(),
df_to_show
.column(COL::METRIC_HUMAN_READABLE_NAME)
.unwrap()
.iter(),
df_to_show.column(COL::METRIC_DESCRIPTION).unwrap().iter(),
df_to_show.column(COL::METRIC_HXL_TAG).unwrap().iter(),
df_to_show.column(COL::GEOMETRY_LEVEL).unwrap().iter(),
) {
let mut table = Table::new();
table
Expand Down
44 changes: 26 additions & 18 deletions src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,11 @@ impl COL {
pub const COUNTRY_ISO2: &'static str = "country_iso2";
pub const COUNTRY_ISO3166_2: &'static str = "country_iso3166_2";

pub const PUBLISHER_ID: &'static str = "data_publisher_id";
pub const PUBLISHER_NAME: &'static str = "data_publisher_name";
pub const PUBLISHER_URL: &'static str = "data_publisher_url";
pub const PUBLISHER_DESCRIPTION: &'static str = "data_publisher_description";
pub const PUBLISHER_COUNTRIES_OF_INTEREST: &'static str =
pub const DATA_PUBLISHER_ID: &'static str = "data_publisher_id";
pub const DATA_PUBLISHER_NAME: &'static str = "data_publisher_name";
pub const DATA_PUBLISHER_URL: &'static str = "data_publisher_url";
pub const DATA_PUBLISHER_DESCRIPTION: &'static str = "data_publisher_description";
pub const DATA_PUBLISHER_COUNTRIES_OF_INTEREST: &'static str =
"data_publisher_countries_of_interest";

pub const GEOMETRY_ID: &'static str = "geometry_id";
Expand All @@ -39,18 +39,26 @@ impl COL {
pub const GEOMETRY_LEVEL: &'static str = "geometry_level";
pub const GEOMETRY_HXL_TAG: &'static str = "geometry_hxl_tag";

pub const SOURCE_ID: &'static str = "source_id";
pub const SOURCE_NAME: &'static str = "source_name";
pub const SOURCE_DATE_PUBLISHED: &'static str = "source_date_published";
pub const SOURCE_REFERENCE_PERIOD_START: &'static str = "source_reference_period_start";
pub const SOURCE_REFERENCE_PERIOD_END: &'static str = "source_reference_period_end";
pub const SOURCE_COLLECTION_PERIOD_START: &'static str = "source_collection_period_start";
pub const SOURCE_COLLECTION_PERIOD_END: &'static str = "source_collection_period_end";
pub const SOURCE_EXPECT_NEXT_UPDATE: &'static str = "source_expect_next_update";
pub const SOURCE_URL: &'static str = "source_url";
pub const SOURCE_DATA_PUBLISHER_ID: &'static str = "source_data_publisher_id";
pub const SOURCE_DESCRIPTION: &'static str = "source_description";
pub const SOURCE_GEOMETRY_METADATA_ID: &'static str = "source_geometry_metadata_id";
pub const SOURCE_DATA_RELEASE_ID: &'static str = "source_data_release_id";
pub const SOURCE_DATA_RELEASE_NAME: &'static str = "source_data_release_name";
pub const SOURCE_DATA_RELEASE_DATE_PUBLISHED: &'static str =
"source_data_release_date_published";
pub const SOURCE_DATA_RELEASE_REFERENCE_PERIOD_START: &'static str =
"source_data_release_reference_period_start";
pub const SOURCE_DATA_RELEASE_REFERENCE_PERIOD_END: &'static str =
"source_data_release_reference_period_end";
pub const SOURCE_DATA_RELEASE_COLLECTION_PERIOD_START: &'static str =
"source_data_release_collection_period_start";
pub const SOURCE_DATA_RELEASE_COLLECTION_PERIOD_END: &'static str =
"source_data_release_collection_period_end";
pub const SOURCE_DATA_RELEASE_EXPECT_NEXT_UPDATE: &'static str =
"source_data_release_expect_next_update";
pub const SOURCE_DATA_RELEASE_URL: &'static str = "source_data_release_url";
pub const SOURCE_DATA_RELEASE_DATA_PUBLISHER_ID: &'static str =
"source_data_release_data_publisher_id";
pub const SOURCE_DATA_RELEASE_DESCRIPTION: &'static str = "source_data_release_description";
pub const SOURCE_DATA_RELEASE_GEOMETRY_METADATA_ID: &'static str =
"source_data_release_geometry_metadata_id";

pub const METRIC_ID: &'static str = "metric_id";
pub const METRIC_HUMAN_READABLE_NAME: &'static str = "metric_human_readable_name";
Expand All @@ -64,7 +72,7 @@ impl COL {
pub const METRIC_PARQUET_MARGIN_OF_ERROR_FILE: &'static str =
"metric_parquet_margin_of_error_file";
pub const METRIC_POTENTIAL_DENOMINATOR_IDS: &'static str = "metric_potential_denominator_ids";
pub const METRIC_PARENT_METRIC_ID: &'static str = "metric_parent_id";
pub const METRIC_PARENT_METRIC_ID: &'static str = "metric_parent_metric_id";
pub const METRIC_SOURCE_DATA_RELEASE_ID: &'static str = "metric_source_data_release_id";
pub const METRIC_SOURCE_DOWNLOAD_URL: &'static str = "metric_source_download_url";
pub const METRIC_SOURCE_ARCHIVE_FILE_PATH: &'static str = "metric_source_archive_file_path";
Expand Down
10 changes: 5 additions & 5 deletions src/metadata.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,21 +93,21 @@ impl Metadata {
.join(
self.source_data_releases.clone().lazy(),
[col(COL::METRIC_SOURCE_DATA_RELEASE_ID)],
[col(COL::SOURCE_ID)],
[col(COL::SOURCE_DATA_RELEASE_ID)],
JoinArgs::new(JoinType::Inner),
)
// Join geometry metadata
.join(
self.geometries.clone().lazy(),
[col(COL::SOURCE_GEOMETRY_METADATA_ID)],
[col(COL::SOURCE_DATA_RELEASE_GEOMETRY_METADATA_ID)],
[col(COL::GEOMETRY_ID)],
JoinArgs::new(JoinType::Inner),
)
// Join data publishers
.join(
self.data_publishers.clone().lazy(),
[col(COL::SOURCE_DATA_PUBLISHER_ID)],
[col(COL::PUBLISHER_ID)],
[col(COL::SOURCE_DATA_RELEASE_DATA_PUBLISHER_ID)],
[col(COL::DATA_PUBLISHER_ID)],
JoinArgs::new(JoinType::Inner),
);
// TODO: Add a country_id column to the metadata, and merge in the countries as well. See
Expand Down Expand Up @@ -255,7 +255,7 @@ mod tests {
#[tokio::test]
async fn country_metadata_should_load() {
let config = Config::default();
let metadata = CountryMetadataLoader::new("be").load(&config).await;
let metadata = CountryMetadataLoader::new("bel").load(&config).await;
println!("{metadata:#?}");
assert!(metadata.is_ok(), "Data should have loaded ok");
}
Expand Down
16 changes: 8 additions & 8 deletions src/search.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,13 +111,13 @@ impl From<SearchText> for Expr {
impl From<YearRange> for Expr {
fn from(value: YearRange) -> Self {
match value {
YearRange::Before(year) => col(COL::SOURCE_REFERENCE_PERIOD_START)
YearRange::Before(year) => col(COL::SOURCE_DATA_RELEASE_REFERENCE_PERIOD_START)
.lt_eq(lit(NaiveDate::from_ymd_opt(year.into(), 12, 31).unwrap())),
YearRange::After(year) => col(COL::SOURCE_REFERENCE_PERIOD_END)
YearRange::After(year) => col(COL::SOURCE_DATA_RELEASE_REFERENCE_PERIOD_END)
.gt_eq(lit(NaiveDate::from_ymd_opt(year.into(), 1, 1).unwrap())),
YearRange::Between(start, end) => {
let start_col = col(COL::SOURCE_REFERENCE_PERIOD_START);
let end_col = col(COL::SOURCE_REFERENCE_PERIOD_END);
let start_col = col(COL::SOURCE_DATA_RELEASE_REFERENCE_PERIOD_START);
let end_col = col(COL::SOURCE_DATA_RELEASE_REFERENCE_PERIOD_END);
let start_date = lit(NaiveDate::from_ymd_opt(start.into(), 1, 1).unwrap());
let end_date = lit(NaiveDate::from_ymd_opt(end.into(), 12, 31).unwrap());
// (start_col <= start_date AND end_col >= start_date)
Expand All @@ -140,13 +140,13 @@ impl From<YearRange> for Expr {

impl From<DataPublisher> for Expr {
fn from(value: DataPublisher) -> Self {
case_insensitive_contains(COL::PUBLISHER_NAME, &value.0)
case_insensitive_contains(COL::DATA_PUBLISHER_NAME, &value.0)
}
}

impl From<SourceDataRelease> for Expr {
fn from(value: SourceDataRelease) -> Self {
case_insensitive_contains(COL::SOURCE_NAME, &value.0)
case_insensitive_contains(COL::SOURCE_DATA_RELEASE_NAME, &value.0)
}
}

Expand Down Expand Up @@ -293,7 +293,7 @@ impl SearchResults {
.select([
col(COL::METRIC_PARQUET_PATH),
col(COL::METRIC_PARQUET_COLUMN_NAME),
col(COL::GEOMETRY_FILENAME_STEM),
col(COL::GEOMETRY_FILEPATH_STEM),
])
.collect()
.unwrap();
Expand All @@ -310,7 +310,7 @@ impl SearchResults {
.into_no_null_iter(),
)
.zip(
df.column(COL::GEOMETRY_FILENAME_STEM)
df.column(COL::GEOMETRY_FILEPATH_STEM)
.unwrap()
.str()
.unwrap()
Expand Down

0 comments on commit 4593779

Please sign in to comment.