Merge pull request #951 from hove-io/NAV-3079_gtfs2ntfs_support_gtfs_…

…deeplinks

#NAV-3079 - Ntfs2gtfs - Add ticketing_deep_links to gtfs export

Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 authors = ["Hove <[email protected]>", "Guillaume Pinot <[email protected]>"]
 name = "transit_model"
-version = "0.62.2"
+version = "0.62.3"
 license = "AGPL-3.0-only"
 description = "Transit data management"
 repository = "https://github.com/hove-io/transit_model"

ntfs2gtfs/tests/fixtures/input_ntfs_with_fare_urls/calendar.txt

@@ -0,0 +1,2 @@
+service_id,monday,tuesday,wednesday,thursday,friday,saturday,sunday,start_date,end_date
+Week,1,1,1,1,1,0,0,20180101,20181231

ntfs2gtfs/tests/fixtures/input_ntfs_with_fare_urls/commercial_modes.txt

@@ -0,0 +1,4 @@
+commercial_mode_id,commercial_mode_name
+Bus,Bus
+Metro,Metro
+RER,Réseau Express Régional (RER)

ntfs2gtfs/tests/fixtures/input_ntfs_with_fare_urls/companies.txt

@@ -0,0 +1,2 @@
+company_id,company_name,company_address,company_url,company_mail,company_phone
+TGC,The Great Company,,,,

ntfs2gtfs/tests/fixtures/input_ntfs_with_fare_urls/contributors.txt

@@ -0,0 +1,2 @@
+contributor_id,contributor_name,contributor_license,contributor_website
+TGC,The Great Contributor,,

ntfs2gtfs/tests/fixtures/input_ntfs_with_fare_urls/datasets.txt

@@ -0,0 +1,2 @@
+dataset_id,contributor_id,dataset_start_date,dataset_end_date,dataset_type,dataset_extrapolation,dataset_desc,dataset_system
+TGDS,TGC,20180101,20181231,,0,,

ntfs2gtfs/tests/fixtures/input_ntfs_with_fare_urls/feed_infos.txt

@@ -0,0 +1,7 @@
+feed_info_param,feed_info_value
+feed_creation_date,20240703
+feed_creation_datetime,2024-07-03T12:49:20+00:00
+feed_creation_time,12:49:20
+feed_end_date,20181231
+feed_start_date,20180101
+ntfs_version,0.15.0

ntfs2gtfs/tests/fixtures/input_ntfs_with_fare_urls/lines.txt

@@ -0,0 +1,4 @@
+line_id,line_code,line_name,forward_line_name,backward_line_name,line_color,line_text_color,line_sort_order,network_id,commercial_mode_id,geometry_id,line_opening_time,line_closing_time
+M1,,Metro 1,,,,,,N1,Metro,,09:00:00,11:10:00
+B42,,Bus 42,,,,,,N2,Bus,,07:00:00,10:20:00
+RERA,,RER A,,,,,,N3,RER,,08:10:00,19:34:00

ntfs2gtfs/tests/fixtures/input_ntfs_with_fare_urls/networks.txt

@@ -0,0 +1,4 @@
+network_id,network_name,network_url,network_timezone,network_lang,network_phone,network_address,network_fare_url,network_sort_order
+N1,Network 1,,,,,,,
+N2,Network 2,,,,,,http://www.network2.com,
+N3,Network 3,,,,,,http://www.network3.com,

ntfs2gtfs/tests/fixtures/input_ntfs_with_fare_urls/physical_modes.txt

@@ -0,0 +1,7 @@
+physical_mode_id,physical_mode_name,co2_emission
+Bus,Bus,132.0
+Metro,Metro,3.0
+RapidTransit,Rapid Transit,7.28
+Bike,Bike,0.0
+BikeSharingService,BikeSharingService,0.0
+Car,Car,184.0

ntfs2gtfs/tests/fixtures/input_ntfs_with_fare_urls/routes.txt

@@ -0,0 +1,7 @@
+route_id,route_name,direction_type,line_id,geometry_id,destination_id
+M1F,Nation - Charles de Gaulle,forward,M1,,CDG
+M1B,Charles de Gaulle - Nation,forward,M1,,NAT
+B42F,Gare de Lyon - Montparnasse,forward,B42,,MTP
+B42B,Montparnasse - Gare de Lyon,forward,B42,,GDL
+RERAF,Nation - La Défense,forward,RERA,,DEF
+RERAB,La Défense - Nation,forward,RERA,,Navitia:MTPZ

ntfs2gtfs/tests/fixtures/input_ntfs_with_fare_urls/stop_times.txt

@@ -0,0 +1,24 @@
+stop_id,trip_id,stop_sequence,arrival_time,departure_time,boarding_duration,alighting_duration,pickup_type,drop_off_type,local_zone_id,stop_headsign,stop_time_id,stop_time_precision
+NATM,M1F1,0,09:00:00,09:00:00,0,0,0,1,,,,0
+GDLM,M1F1,1,09:10:00,09:10:00,0,0,3,3,,,,0
+CHAM,M1F1,2,09:20:00,09:20:00,0,0,0,0,,,,0
+CDGM,M1F1,3,09:40:00,09:40:00,0,0,1,0,,,,0
+CDGM,M1B1,6,10:40:00,10:40:00,0,0,0,1,,,,0
+CHAM,M1B1,7,10:50:00,10:50:00,0,0,0,0,,,,0
+GDLM,M1B1,8,11:00:00,11:00:00,0,0,0,0,,,,0
+NATM,M1B1,9,11:10:00,11:10:00,0,0,1,0,,,,0
+GDLB,B42F1,10,10:10:00,10:10:00,0,0,0,1,,,,0
+MTPB,B42F1,20,10:20:00,10:20:00,0,0,1,0,,,,0
+MTPB,B42B1,20,07:00:00,07:00:00,0,0,0,1,,,,0
+GDLB,B42B1,30,07:10:00,07:10:00,0,0,1,0,,,,0
+NATR,RERAF1,1,08:09:00,08:10:00,0,0,0,1,,,,0
+GDLR,RERAF1,2,08:14:00,08:15:00,0,0,0,0,,,,0
+CDGR,RERAF1,3,08:19:00,08:20:00,0,0,0,0,,,,0
+DEFR,RERAF1,5,08:24:00,08:25:00,0,0,1,0,,,,0
+DEFR,RERAB1,5,09:24:00,09:25:00,0,0,0,1,,,,2
+CDGR,RERAB1,8,09:39:00,09:40:00,0,0,0,0,,,,0
+GDLR,RERAB1,13,09:44:00,09:45:00,0,0,0,0,,,,0
+NATR,RERAB1,21,09:49:00,09:50:00,0,0,0,0,,,,0
+MTPZ,RERAB1,50,19:24:00,19:25:00,0,0,0,0,,,,2
+CDGZ,RERAB1,51,19:26:00,19:27:00,0,0,0,0,,,,0
+MTPZ,RERAB1,52,19:34:00,19:35:00,0,0,1,0,,,,2

ntfs2gtfs/tests/fixtures/input_ntfs_with_fare_urls/stops.txt

@@ -0,0 +1,21 @@
+stop_id,stop_name,stop_code,visible,fare_zone_id,stop_lon,stop_lat,location_type,parent_station,stop_timezone,geometry_id,equipment_id,level_id,platform_code,address_id
+GDLR,Gare de Lyon (RER),,1,,2.372987,48.844746,0,GDL,,,,,,
+GDLM,Gare de Lyon (Metro),,1,,2.372987,48.844746,0,GDL,,,,,,
+GDLB,Gare de Lyon (Bus),,1,,2.372987,48.844746,0,GDL,,,,,,
+NATR,Nation (RER),,1,,2.396497,48.84849,0,NAT,,,,,,
+NATM,Nation (Metro),,1,,2.396497,48.84849,0,NAT,,,,,,
+CDGR,Charles de Gaulle (RER),,1,,2.295354,48.873965,0,CDG,,,,,,
+CDGM,Charles de Gaulle (Metro),,1,,2.795354,48.973965,0,CDG,,,,,,
+DEFR,La Défense (RER),,1,,2.238964,48.891737,0,DEF,,,,,,
+CHAM,Châtelet (Metro),,1,,2.348145,48.858137,0,CHA,,,,,,
+MTPB,Montparnasse (Bus),,1,,2.321783,48.842481,0,MTP,,,,,,
+MTPZ,Montparnasse Zone,,1,,2.321783,48.842481,2,Navitia:MTPZ,,,,,,
+CDGZ,Charles de Gaulle Zone,,1,,2.321783,48.842481,2,Navitia:CDGZ,,,,,,
+GDL,Gare de Lyon,,1,,2.372987,48.844746,1,,,,,,,
+NAT,Nation,,1,,2.396497,48.84849,1,,,,,,,
+CDG,Charles de Gaulle,,1,,2.295354,48.873965,1,,,,,,,
+DEF,La Défense,,1,,2.238964,48.891737,1,,,,,,,
+CHA,Châtelet,,1,,2.348145,48.858137,1,,,,,,,
+MTP,Montparnasse,,1,,2.321783,48.842481,1,,,,,,,
+Navitia:MTPZ,Montparnasse Zone,,0,,2.321783,48.842481,1,,,,,,,
+Navitia:CDGZ,Charles de Gaulle Zone,,0,,2.321783,48.842481,1,,,,,,,

ntfs2gtfs/tests/fixtures/input_ntfs_with_fare_urls/transfers.txt

@@ -0,0 +1,27 @@
+from_stop_id,to_stop_id,min_transfer_time,real_min_transfer_time,equipment_id
+GDLM,GDLM,0,60,
+GDLR,GDLR,0,60,
+CDGZ,MTPB,0,60,
+GDLB,GDLM,0,60,
+GDLM,GDLB,0,60,
+GDLM,GDLR,0,60,
+GDLR,GDLB,0,60,
+GDLB,GDLR,0,60,
+MTPZ,CDGZ,0,60,
+MTPB,MTPB,0,60,
+MTPZ,MTPZ,0,60,
+CDGZ,MTPZ,0,60,
+CDGM,CDGM,0,60,
+NATM,NATM,0,60,
+MTPB,CDGZ,0,60,
+MTPB,MTPZ,0,60,
+NATR,NATR,0,60,
+DEFR,DEFR,0,60,
+CHAM,CHAM,0,60,
+NATM,NATR,0,60,
+GDLB,GDLB,0,60,
+NATR,NATM,0,60,
+GDLR,GDLM,0,60,
+CDGR,CDGR,0,60,
+MTPZ,MTPB,0,60,
+CDGZ,CDGZ,0,60,

ntfs2gtfs/tests/fixtures/input_ntfs_with_fare_urls/trips.txt

@@ -0,0 +1,7 @@
+trip_id,route_id,physical_mode_id,dataset_id,service_id,trip_headsign,trip_short_name,block_id,company_id,trip_property_id,geometry_id,journey_pattern_id
+M1F1,M1F,Metro,TGDS,Week,Charles de Gaulle (Metro),,,TGC,,,
+M1B1,M1B,Metro,TGDS,Week,Nation (Metro),,,TGC,,,
+B42F1,B42F,Bus,TGDS,Week,Montparnasse (Bus),,,TGC,,,
+B42B1,B42B,Bus,TGDS,Week,Gare de Lyon (Bus),,,TGC,,,
+RERAF1,RERAF,RapidTransit,TGDS,Week,La Défense (RER),,,TGC,,,
+RERAB1,RERAB,Bus,TGDS,Week,Montparnasse Zone,,,TGC,,,

ntfs2gtfs/tests/fixtures/output/agency.txt

@@ -1,2 +1,2 @@
-agency_id,agency_name,agency_url,agency_timezone,agency_lang,agency_phone,agency_email,agency_fare_url
-network:kept,The Great Network,https://hove.com,Europe/Paris,,,,https://hove.com/tickets
+agency_id,agency_name,agency_url,agency_timezone,agency_lang,agency_phone,agency_email,agency_fare_url,ticketing_deep_link_id
+network:kept,The Great Network,https://hove.com,Europe/Paris,,,,https://hove.com/tickets,ticketing_deep_link:1

ntfs2gtfs/tests/fixtures/output/ticketing_deep_links.txt

@@ -0,0 +1,2 @@
+ticketing_deep_link_id,web_url,android_intent_uri,ios_universal_link_url
+ticketing_deep_link:1,https://hove.com/tickets,https://hove.com/tickets,https://hove.com/tickets

ntfs2gtfs/tests/fixtures/output_gtfs_with_fare_url_deeplinks/agency.txt

@@ -0,0 +1,4 @@
+agency_id,agency_name,agency_url,agency_timezone,agency_lang,agency_phone,agency_email,agency_fare_url,ticketing_deep_link_id
+N1,Network 1,http://www.navitia.io/,Europe/Paris,,,,,
+N2,Network 2,http://www.navitia.io/,Europe/Paris,,,,http://www.network2.com,ticketing_deep_link:1
+N3,Network 3,http://www.navitia.io/,Europe/Paris,,,,http://www.network3.com,ticketing_deep_link:2

ntfs2gtfs/tests/fixtures/output_gtfs_with_fare_url_deeplinks/ticketing_deep_links.txt

@@ -0,0 +1,3 @@
+ticketing_deep_link_id,web_url,android_intent_uri,ios_universal_link_url
+ticketing_deep_link:1,http://www.network2.com,http://www.network2.com,http://www.network2.com
+ticketing_deep_link:2,http://www.network3.com,http://www.network3.com,http://www.network3.com

ntfs2gtfs/tests/ntfs2gtfs.rs

@@ -176,3 +176,21 @@ fn test_ntfs2gtfs_split_route_by_mode_extended() {
         "./tests/fixtures/output_split_route_by_mode_extended",
     );
 }
+
+#[test]
+fn test_ntfs2gtfs_with_fare_urls_and_deeplinks() {
+    let output_dir = TempDir::new().expect("create temp dir failed");
+    Command::cargo_bin("ntfs2gtfs")
+        .expect("Failed to find binary 'ntfs2gtfs'")
+        .arg("--input")
+        .arg("tests/fixtures/input_ntfs_with_fare_urls")
+        .arg("--output")
+        .arg(output_dir.path().to_str().unwrap())
+        .assert()
+        .success();
+    compare_output_dir_with_expected(
+        output_dir,
+        Some(vec!["agency.txt", "ticketing_deep_links.txt"]),
+        "./tests/fixtures/output_gtfs_with_fare_url_deeplinks",
+    );
+}

src/gtfs/mod.rs

@@ -21,7 +21,7 @@ use crate::{
     calendars::{manage_calendars, write_calendar_dates},
     file_handler::{FileHandler, PathFileHandler, ZipHandler},
     model::{Collections, Model},
-    objects::{self, Availability, Contributor, Dataset, StopType, Time},
+    objects::{self, Availability, Contributor, Dataset, Network, StopType, Time},
     parser::read_opt_collection,
     serde_utils::*,
     utils::*,
@@ -31,7 +31,11 @@ use anyhow::{anyhow, Context};
 use chrono_tz::Tz;
 use derivative::Derivative;
 use serde::{Deserialize, Serialize};
-use std::{collections::BTreeMap, fmt, path::Path};
+use std::{
+    collections::{BTreeMap, BTreeSet, HashMap},
+    fmt,
+    path::Path,
+};
 
 use tracing::info;
 use typed_index_collection::CollectionWithId;
@@ -60,6 +64,9 @@ struct Agency {
     email: Option<String>,
     #[serde(rename = "agency_fare_url")]
     fare_url: Option<String>,
+    // Will not export attribute (and therefore csv column) if all values ​​are None
+    #[serde(skip_serializing_if = "Option::is_none")]
+    ticketing_deep_link_id: Option<String>,
 }
 
 impl<'a> From<&'a objects::Network> for Agency {
@@ -76,6 +83,7 @@ impl<'a> From<&'a objects::Network> for Agency {
             phone: obj.phone.clone(),
             email: None,
             fare_url: obj.fare_url.clone(),
+            ticketing_deep_link_id: None,
         }
     }
 }
@@ -264,6 +272,17 @@ struct Shape {
     sequence: u32,
 }
 
+#[derive(Serialize, Debug)]
+struct TicketingDeepLink {
+    #[serde(rename = "ticketing_deep_link_id")]
+    id: String,
+    web_url: Option<String>,
+    android_intent_uri: Option<String>,
+    ios_universal_link_url: Option<String>,
+}
+
+type TicketingDeepLinks = HashMap<String, TicketingDeepLink>;
+
 ///parameters consolidation
 #[derive(Default)]
 pub struct Configuration {
@@ -628,6 +647,27 @@ fn to_gtfs_extended_value(route_type: &RouteType) -> String {
     }
 }
 
+fn get_ticketing_deep_links(networks: &CollectionWithId<Network>) -> TicketingDeepLinks {
+    networks
+        .values()
+        .filter_map(|n| n.fare_url.clone())
+        .collect::<BTreeSet<_>>()
+        .iter()
+        .enumerate()
+        .map(|(i, fare_url)| {
+            (
+                fare_url.clone(),
+                TicketingDeepLink {
+                    id: format!("ticketing_deep_link:{}", i + 1),
+                    web_url: Some(fare_url.clone()),
+                    android_intent_uri: Some(fare_url.clone()),
+                    ios_universal_link_url: Some(fare_url.clone()),
+                },
+            )
+        })
+        .collect::<TicketingDeepLinks>()
+}
+
 fn ser_from_route_type_extended<S>(r: &RouteType, serializer: S) -> Result<S::Ok, S::Error>
 where
     S: serde::Serializer,
@@ -643,8 +683,10 @@ pub fn write<P: AsRef<Path>>(model: Model, path: P, extend_route_type: bool) ->
     std::fs::create_dir_all(path)?;
     info!("Writing GTFS to {:?}", path);
 
+    let ticketing_deep_links = get_ticketing_deep_links(&model.networks);
     write::write_transfers(path, &model.transfers)?;
-    write::write_agencies(path, &model.networks)?;
+    write::write_ticketing_deep_links(path, &ticketing_deep_links)?;
+    write::write_agencies(path, &model.networks, &ticketing_deep_links)?;
     write_calendar_dates(path, &model.calendars)?;
     write::write_stops(
         path,

src/gtfs/write.rs

@@ -13,8 +13,8 @@
 // along with this program. If not, see <https://www.gnu.org/licenses/>
 
 use super::{
-    Agency, DirectionType, Route, RouteType, Shape, Stop, StopLocationType, StopTime, Transfer,
-    Trip,
+    Agency, DirectionType, Route, RouteType, Shape, Stop, StopLocationType, StopTime,
+    TicketingDeepLinks, Transfer, Trip,
 };
 use crate::gtfs::ExtendedRoute;
 use crate::model::{GetCorresponding, Model};
@@ -55,13 +55,29 @@ pub fn write_transfers(path: &path::Path, transfers: &Collection<NtfsTransfer>)
 pub fn write_agencies(
     path: &path::Path,
     networks: &CollectionWithId<objects::Network>,
+    ticketing_deep_links: &TicketingDeepLinks,
 ) -> Result<()> {
     info!("Writing agency.txt");
     let path = path.join("agency.txt");
     let mut wtr =
         csv::Writer::from_path(&path).with_context(|| format!("Error reading {:?}", path))?;
-    for n in networks.values() {
-        wtr.serialize(Agency::from(n))
+    for network in networks.values() {
+        let mut agency = Agency::from(network);
+        if !ticketing_deep_links.is_empty() {
+            agency.ticketing_deep_link_id = network
+                .fare_url
+                .as_ref()
+                .and_then(|fare_url| {
+                    ticketing_deep_links
+                        .get(fare_url)
+                        .map(|ticketing_deep_link| ticketing_deep_link.id.clone())
+                })
+                .or_else(|| Some(String::new()));
+            // If there is at least one ticketing_deep_link_id then the other csv columns cannot be set to None.
+            // See struct Agency -> skip_serializing_if on ticketing_deep_link_id
+            // Since the number of serialized columns must be the same, agencies without ticketing_deep_link_id must be set to empty string
+        }
+        wtr.serialize(agency)
             .with_context(|| format!("Error reading {:?}", path))?;
     }
 
@@ -71,6 +87,26 @@ pub fn write_agencies(
     Ok(())
 }
 
+pub fn write_ticketing_deep_links(
+    path: &path::Path,
+    ticketing_deep_links: &TicketingDeepLinks,
+) -> Result<()> {
+    if !ticketing_deep_links.is_empty() {
+        let file_name = "ticketing_deep_links.txt";
+        info!("Writing {}", file_name);
+        let path = path.join(file_name);
+        let mut wtr =
+            csv::Writer::from_path(&path).with_context(|| format!("Error reading {:?}", path))?;
+        for tdl in ticketing_deep_links.values() {
+            wtr.serialize(tdl)
+                .with_context(|| format!("Error serializing {:?}", path))?;
+        }
+        wtr.flush()
+            .with_context(|| format!("Error reading {:?}", path))?;
+    }
+    Ok(())
+}
+
 /// get the first comment ordered by name
 fn get_first_comment_name<T: objects::CommentLinks>(
     obj: &T,
@@ -545,6 +581,7 @@ mod tests {
             phone: Some("0123456789".to_string()),
             email: None,
             fare_url: Some("http://www.vianavigo.com/tickets".to_string()),
+            ticketing_deep_link_id: None,
         };
 
         assert_eq!(expected_agency, agency);
@@ -574,6 +611,7 @@ mod tests {
             phone: None,
             email: None,
             fare_url: None,
+            ticketing_deep_link_id: None,
         };
 
         assert_eq!(expected_agency, agency);