From 7bc7a9e40b4ff7e1ac3462a69de6dec2c009d7c7 Mon Sep 17 00:00:00 2001 From: heidebritta Date: Fri, 6 Aug 2021 14:06:24 -0400 Subject: [PATCH] Update transformations to clean up metrics, log statements and consolidate service checks so that one file can check both current and future service. --- .../impl/CheckForFutureService.java | 16 ++ .../impl/VerifyFutureRouteService.java | 22 ++- .../impl/VerifyReferenceService.java | 25 +-- .../updates/VerifyRouteService.java | 148 ++++++++++-------- 4 files changed, 125 insertions(+), 86 deletions(-) diff --git a/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/CheckForFutureService.java b/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/CheckForFutureService.java index 7ba43ddc9..5e734168f 100644 --- a/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/CheckForFutureService.java +++ b/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/CheckForFutureService.java @@ -29,6 +29,10 @@ import java.util.Calendar; import java.util.Date; +/* Checks the numbers of Trips with service today and next four days + * Metrics are logged and published to AWS + * can be used for Bus or Subway + */ public class CheckForFutureService implements GtfsTransformStrategy { private final Logger _log = LoggerFactory.getLogger(CheckForFutureService.class); @@ -41,9 +45,11 @@ public String getName() { @Override public void run(TransformContext context, GtfsMutableRelationalDao dao) { + int tripsToday = 0; int tripsTomorrow = 0; int tripsNextDay = 0; int tripsDayAfterNext = 0; + Date today = removeTime(new Date()); Date tomorrow = removeTime(addDays(new Date(), 1)); Date nextDay = removeTime(addDays(new Date(), 2)); Date dayAfterNext = removeTime(addDays(new Date(), 3)); @@ -53,15 +59,25 @@ public void run(TransformContext context, GtfsMutableRelationalDao dao) { String agency = dao.getAllAgencies().iterator().next().getId(); String agencyName = dao.getAllAgencies().iterator().next().getName(); + tripsToday = hasServiceForDate(dao, today); tripsTomorrow = hasServiceForDate(dao, tomorrow); tripsNextDay = hasServiceForDate(dao, nextDay); tripsDayAfterNext = hasServiceForDate(dao,dayAfterNext); + _log.info("Feed for metrics: {}, agency id: {}", feed, agencyName); + es.publishMetric(CloudContextService.getNamespace(), "TripsToday", "feed", feed, tripsToday); es.publishMetric(CloudContextService.getNamespace(), "TripsTomorrow", "feed", feed, tripsTomorrow); es.publishMetric(CloudContextService.getNamespace(), "TripsIn2Days", "feed", feed, tripsNextDay); es.publishMetric(CloudContextService.getNamespace(), "TripsIn3Days", "feed", feed, tripsDayAfterNext); + _log.info("TripsToday: {}, feed: {}, namespace: {}", tripsToday, feed, CloudContextService.getNamespace()); + _log.info("TripsTomorrow: {}, feed: {}, namespace: {}", tripsTomorrow, feed, CloudContextService.getNamespace()); + _log.info("TripsIn2Days: {}, feed: {}, namespace: {}", tripsNextDay, feed, CloudContextService.getNamespace()); + _log.info("TripsIn3Days: {}, feed: {}, namespace: {}", tripsDayAfterNext, feed, CloudContextService.getNamespace()); + if (tripsToday == 0) { + _log.error("Agency {} {} is missing service for today {}", agency, agencyName, tomorrow); + } if (tripsTomorrow == 0) { _log.error("Agency {} {} is missing service for tomorrow {}", agency, agencyName, tomorrow); } diff --git a/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/VerifyFutureRouteService.java b/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/VerifyFutureRouteService.java index f249f1a46..e9c29ead7 100644 --- a/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/VerifyFutureRouteService.java +++ b/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/VerifyFutureRouteService.java @@ -39,6 +39,10 @@ import java.util.*; import java.util.Set; +/* Checks the numbers of Routes running on two input GTFS files and compares them. + * Used for comparing ATIS to Reference GTFS and reporting differences of ATIS routes that are missing service + * Looks at Routes running today through next 3 days + */ public class VerifyFutureRouteService implements GtfsTransformStrategy { private final Logger _log = LoggerFactory.getLogger(VerifyFutureRouteService.class); @@ -81,27 +85,33 @@ public void run(TransformContext context, GtfsMutableRelationalDao dao) { CalendarService refCalendarService = CalendarServiceDataFactoryImpl.createService(reference); String feed = CloudContextService.getLikelyFeedName(dao); ExternalServices es = new ExternalServicesBridgeFactory().getExternalServices(); + String agencyName = dao.getAllAgencies().iterator().next().getName(); + int[] tripsToday; int[] tripsTomorrow; int[] tripsNextDay; int[] tripsDayAfterNext; + Date today = removeTime(new Date()); Date tomorrow = removeTime(addDays(new Date(), 1)); Date nextDay = removeTime(addDays(new Date(), 2)); Date dayAfterNext = removeTime(addDays(new Date(), 3)); + tripsToday = hasRouteServiceForDate(dao, reference, refCalendarService, today, problemRoutes); tripsTomorrow = hasRouteServiceForDate(dao, reference, refCalendarService, tomorrow, problemRoutes); tripsNextDay = hasRouteServiceForDate(dao, reference, refCalendarService, nextDay, problemRoutes); tripsDayAfterNext = hasRouteServiceForDate(dao, reference, refCalendarService, dayAfterNext, problemRoutes); - _log.info("Active routes {}: {}, {}: {}, {}: {}", - tomorrow, tripsTomorrow, nextDay, tripsNextDay, dayAfterNext, tripsDayAfterNext); + _log.info("Feed for metrics: {}, agency name: {}", feed, agencyName); + _log.info("Active routes {}: {}, {}: {}, {}: {}, {}: {}", + today, tripsToday[ACTIVE_ROUTES], tomorrow, tripsTomorrow[ACTIVE_ROUTES], nextDay, tripsNextDay[ACTIVE_ROUTES], dayAfterNext, tripsDayAfterNext[ACTIVE_ROUTES]); + es.publishMetric(CloudContextService.getNamespace(), "RoutesContainingTripsToday", "feed", feed, tripsToday[ACTIVE_ROUTES]); + es.publishMetric(CloudContextService.getNamespace(), "RoutesNoTripsInAtisButInRefToday", "feed", feed, tripsToday[ALARMING_ROUTES]); es.publishMetric(CloudContextService.getNamespace(), "RoutesContainingTripsTomorrow", "feed", feed, tripsTomorrow[ACTIVE_ROUTES]); - es.publishMetric(CloudContextService.getNamespace(), "RoutesMissingTripsFromAtisButInRefTomorrow", "feed", feed, tripsTomorrow[ALARMING_ROUTES]); + es.publishMetric(CloudContextService.getNamespace(), "RoutesNoTripsInAtisButInRefTomorrow", "feed", feed, tripsTomorrow[ALARMING_ROUTES]); es.publishMetric(CloudContextService.getNamespace(), "RoutesContainingTripsIn2Days", "feed", feed, tripsNextDay[ACTIVE_ROUTES]); - es.publishMetric(CloudContextService.getNamespace(), "RoutesMissingTripsFromAtisButInRefIn2Days", "feed", feed, tripsNextDay[ALARMING_ROUTES]); + es.publishMetric(CloudContextService.getNamespace(), "RoutesNoTripsInAtisButInRefIn2Days", "feed", feed, tripsNextDay[ALARMING_ROUTES]); es.publishMetric(CloudContextService.getNamespace(), "RoutesContainingTripsIn3Days", "feed", feed, tripsDayAfterNext[ACTIVE_ROUTES]); - es.publishMetric(CloudContextService.getNamespace(), "RoutesMissingTripsFromAtisButInRefIn3Days", "feed", feed, tripsDayAfterNext[ALARMING_ROUTES]); - + es.publishMetric(CloudContextService.getNamespace(), "RoutesNoTripsInAtisButInRefIn3Days", "feed", feed, tripsDayAfterNext[ALARMING_ROUTES]); } private int[] hasRouteServiceForDate(GtfsMutableRelationalDao dao, GtfsMutableRelationalDao reference, diff --git a/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/VerifyReferenceService.java b/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/VerifyReferenceService.java index 068f79cb0..6a1d6f28c 100644 --- a/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/VerifyReferenceService.java +++ b/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/VerifyReferenceService.java @@ -36,8 +36,9 @@ import java.io.InputStream; import java.util.*; -/* Check reference service against ATIS service and if there is service in ATIS -that isn't in reference send a notification +/* Checks the numbers of Routes running on two input GTFS files and compares them. + * Used for comparing ATIS to Reference GTFS and reporting differences of Reference routes that are missing service + * Looks at Routes running today through next 3 days */ public class VerifyReferenceService implements GtfsTransformStrategy { private final int ACTIVE_ROUTES = 0; @@ -54,7 +55,7 @@ public String getName() { public void run(TransformContext context, GtfsMutableRelationalDao dao) { GtfsMutableRelationalDao reference = (GtfsMutableRelationalDao) context.getReferenceReader().getEntityStore(); CalendarService refCalendarService = CalendarServiceDataFactoryImpl.createService(reference); - String feed = CloudContextService.getLikelyFeedName(dao); + String feed = CloudContextService.getLikelyFeedName(reference); ExternalServices es = new ExternalServicesBridgeFactory().getExternalServices(); Collection problemRoutes; @@ -76,7 +77,6 @@ public void run(TransformContext context, GtfsMutableRelationalDao dao) { } - int[] tripsToday; int[] tripsTomorrow; int[] tripsNextDay; @@ -91,16 +91,17 @@ public void run(TransformContext context, GtfsMutableRelationalDao dao) { tripsNextDay = hasRouteServiceForDate(dao, reference, refCalendarService, nextDay, problemRoutes); tripsDayAfterNext = hasRouteServiceForDate(dao, reference, refCalendarService, dayAfterNext, problemRoutes); + _log.info("Feed for metrics: {}", feed); _log.info("Active routes {}: {}, {}: {}, {}: {}, {}: {}", today, tripsToday, tomorrow, tripsTomorrow, nextDay, tripsNextDay, dayAfterNext, tripsDayAfterNext); - es.publishMetric(CloudContextService.getNamespace(), "RoutesContainingTripsToday", "feed", feed, tripsTomorrow[ACTIVE_ROUTES]); - es.publishMetric(CloudContextService.getNamespace(), "RoutesMissingTripsFromRefButInAtisToday", "feed", feed, tripsTomorrow[ALARMING_ROUTES]); - es.publishMetric(CloudContextService.getNamespace(), "RoutesContainingTripsTomorrow", "feed", feed, tripsTomorrow[ACTIVE_ROUTES]); - es.publishMetric(CloudContextService.getNamespace(), "RoutesMissingTripsFromRefButInAtisTomorrow", "feed", feed, tripsTomorrow[ALARMING_ROUTES]); - es.publishMetric(CloudContextService.getNamespace(), "RoutesContainingTripsIn2Days", "feed", feed, tripsNextDay[ACTIVE_ROUTES]); - es.publishMetric(CloudContextService.getNamespace(), "RoutesMissingTripsFromRefButInAtisIn2Days", "feed", feed, tripsNextDay[ALARMING_ROUTES]); - es.publishMetric(CloudContextService.getNamespace(), "RoutesContainingTripsIn3Days", "feed", feed, tripsDayAfterNext[ACTIVE_ROUTES]); - es.publishMetric(CloudContextService.getNamespace(), "RoutesMissingTripsFromRefButInAtisIn3Days", "feed", feed, tripsDayAfterNext[ALARMING_ROUTES]); + es.publishMetric(CloudContextService.getNamespace(), "RefRoutesContainingTripsToday", "feed", feed, tripsTomorrow[ACTIVE_ROUTES]); + es.publishMetric(CloudContextService.getNamespace(), "RefRoutesMissingTripsFromRefButInAtisToday", "feed", feed, tripsTomorrow[ALARMING_ROUTES]); + es.publishMetric(CloudContextService.getNamespace(), "RefRoutesContainingTripsTomorrow", "feed", feed, tripsTomorrow[ACTIVE_ROUTES]); + es.publishMetric(CloudContextService.getNamespace(), "RefRoutesMissingTripsFromRefButInAtisTomorrow", "feed", feed, tripsTomorrow[ALARMING_ROUTES]); + es.publishMetric(CloudContextService.getNamespace(), "RefRoutesContainingTripsIn2Days", "feed", feed, tripsNextDay[ACTIVE_ROUTES]); + es.publishMetric(CloudContextService.getNamespace(), "RefRoutesMissingTripsFromRefButInAtisIn2Days", "feed", feed, tripsNextDay[ALARMING_ROUTES]); + es.publishMetric(CloudContextService.getNamespace(), "RefRoutesContainingTripsIn3Days", "feed", feed, tripsDayAfterNext[ACTIVE_ROUTES]); + es.publishMetric(CloudContextService.getNamespace(), "RefRoutesMissingTripsFromRefButInAtisIn3Days", "feed", feed, tripsDayAfterNext[ALARMING_ROUTES]); } int[] hasRouteServiceForDate(GtfsMutableRelationalDao dao, GtfsMutableRelationalDao reference, diff --git a/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/updates/VerifyRouteService.java b/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/updates/VerifyRouteService.java index af55c0bc9..edc2825a9 100644 --- a/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/updates/VerifyRouteService.java +++ b/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/updates/VerifyRouteService.java @@ -29,11 +29,16 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.text.DateFormat; +import java.text.SimpleDateFormat; import java.util.*; -/* -Used for subway service as it has specific logic for the subway routes and the express routes - */ +/* checks number of routes on input GTFS and checks to see if there is service on the route + * sends metrics for number of active routes each day + * sends metrics for number of trips on each active route + * logs if a route has no service + * */ + public class VerifyRouteService implements GtfsTransformStrategy { private final Logger _log = LoggerFactory.getLogger(VerifyRouteService.class); @@ -45,88 +50,83 @@ public String getName() { @Override public void run(TransformContext context, GtfsMutableRelationalDao dao) { - GtfsMutableRelationalDao reference = (GtfsMutableRelationalDao) context.getReferenceReader().getEntityStore(); String feed = CloudContextService.getLikelyFeedName(dao); ExternalServices es = new ExternalServicesBridgeFactory().getExternalServices(); - CalendarService refCalendarService = CalendarServiceDataFactoryImpl.createService(reference); - AgencyAndId refAgencyAndId = reference.getAllTrips().iterator().next().getId(); - - int curSerRoute = 0; - int alarmingRoutes = 0; - boolean missingRoute = false; - boolean missingService = false; Date today = removeTime(new Date()); - //list of all routes in ATIS - Set ATISrouteIds = new HashSet<>(); + Date tomorrow = removeTime(addDays(new Date(), 1)); + Date nextDay = removeTime(addDays(new Date(), 2)); + Date dayAfterNext = removeTime(addDays(new Date(), 3)); + + int numRoutesToday = hasRouteServiceForDate(dao, today, es); + if (!isWeekend(today)) { + es.publishMetric(CloudContextService.getNamespace(), "WeekdayActiveSubwayRoutesToday", "feed", feed, numRoutesToday); + } + int numRoutesTommorrow = hasRouteServiceForDate(dao, tomorrow, es); + if (!isWeekend(tomorrow)) { + es.publishMetric(CloudContextService.getNamespace(), "WeekdayActiveSubwayRoutesTomorrow", "feed", feed, numRoutesTommorrow); + } + int numRoutesIn2Days = hasRouteServiceForDate(dao, nextDay, es); + if (!isWeekend(nextDay)) { + es.publishMetric(CloudContextService.getNamespace(), "WeekdayActiveSubwayRoutesIn2Days", "feed", feed, numRoutesIn2Days); + } + int numRoutesIn3Days = hasRouteServiceForDate(dao, dayAfterNext, es); + if (!isWeekend(dayAfterNext)) { + es.publishMetric(CloudContextService.getNamespace(), "WeekdayActiveSubwayRoutesIn3Days", "feed", feed, numRoutesIn3Days); + } + + _log.info("Feed for metrics: {}", feed); + _log.info("Active routes: {}: {}; {}: {}; {}: {}; {}: {}", + formatDate(today), numRoutesToday, formatDate(tomorrow), numRoutesTommorrow, formatDate(nextDay), numRoutesIn2Days, formatDate(dayAfterNext), numRoutesIn3Days); + es.publishMetric(CloudContextService.getNamespace(), "RoutesContainingTripsToday", "feed", feed, numRoutesToday); + es.publishMetric(CloudContextService.getNamespace(), "RoutesContainingTripsTomorrow", "feed", feed, numRoutesTommorrow); + es.publishMetric(CloudContextService.getNamespace(), "RoutesContainingTripsIn2Days", "feed", feed, numRoutesIn2Days); + es.publishMetric(CloudContextService.getNamespace(), "RoutesContainingTripsIn3Days", "feed", feed, numRoutesIn3Days); + + if (numRoutesToday < 3) { + throw new IllegalStateException( + "Route service missing in agency: " + dao.getAllAgencies().iterator().next()); + } + } + + private int hasRouteServiceForDate(GtfsMutableRelationalDao dao, Date testDate, ExternalServices es) { + int numTripsOnDate = 0; + int activeRoutes = 0; //check for route specific current service for (Route route : dao.getAllRoutes()) { - if (route.getId().getId().length() > 2) { - ATISrouteIds.add(route.getId().getId().substring(0,2)); - _log.info("Adding route: {}", route.getId().getId().substring(0,2)); - } else { - ATISrouteIds.add(route.getId().getId()); - _log.info("Adding route: {}", route.getId().getId()); - } - curSerRoute = 0; + numTripsOnDate = 0; triploop: - for (Trip trip1 : dao.getTripsForRoute(route)) { - for (ServiceCalendarDate calDate : dao.getCalendarDatesForServiceId(trip1.getServiceId())) { + for (Trip trip : dao.getTripsForRoute(route)) { + for (ServiceCalendarDate calDate : dao.getCalendarDatesForServiceId(trip.getServiceId())) { Date date = constructDate(calDate.getDate()); - if (calDate.getExceptionType() == 1 && date.equals(today)) { - _log.info("ATIS has current service for route: {}", route.getId().getId()); - curSerRoute++; - break triploop; - } - } - } - if (curSerRoute == 0) { - _log.error("No current service for {}", route.getId().getId()); - //if there is no current service, check that it should have service - //there are certain routes that don't run on the weekend or won't have service in reference - ServiceDate sToday = createServiceDate(today); - Route refRoute = reference.getRouteForId(new AgencyAndId(refAgencyAndId.getAgencyId(), route.getId().getId())); - reftriploop: - for (Trip refTrip : reference.getTripsForRoute(refRoute)) { - Set activeDates = refCalendarService.getServiceDatesForServiceId(refTrip.getServiceId()); - if (activeDates.contains(sToday)) { - //ignore express routes, MOTP-1184 MOTP-1259 - if (!route.getId().getId().contains("X")) { - missingService = true; - _log.info("Reference has service for this route today but ATIS has none: {}", route.getId()); - alarmingRoutes++; - } - break reftriploop; + if (calDate.getExceptionType() == 1 && date.equals(testDate)) { + numTripsOnDate++; } } } - } - - int referenceRoutesInAtis = 0; - int referenceRoutesNotInAtis = 0; - //check that every route in reference GTFS is also in ATIS gtfs - for (Route route : reference.getAllRoutes()) { - if (!ATISrouteIds.contains(route.getId().getId())) { - missingRoute = true; - _log.error("ATIS GTFS missing route {}", route.getId()); - referenceRoutesNotInAtis++; + if (numTripsOnDate == 0) { + _log.info("No service for {} on {}", route.getId().getId(), testDate); } - else{ - referenceRoutesInAtis++; + else { + activeRoutes++; + _log.info("Route: {} {} Number of trips: {}", route.getId().getId(), formatDate(testDate), numTripsOnDate); + //this metric is published each time any route is Active (has trips on given day) + String metricName = "TripsOnRoute" + route.getId().getId(); + es.publishMetric(CloudContextService.getNamespace(), metricName, "day", formatDate(testDate), numTripsOnDate); } } + return activeRoutes; + } - es.publishMetric(CloudContextService.getNamespace(), "RoutesMissingTripsFromAtisButInRefToday", "feed", feed, alarmingRoutes); - es.publishMetric(CloudContextService.getNamespace(), "RoutesContainingTripsToday", "feed", feed, curSerRoute); - es.publishMetric(CloudContextService.getNamespace(), "RoutesInReferenceButNotAtis", "feed", feed, referenceRoutesNotInAtis); - es.publishMetric(CloudContextService.getNamespace(), "RoutesInReferenceAndAtis", "feed", feed, referenceRoutesInAtis); - - - if (missingService || missingRoute) { - throw new IllegalStateException( - "Route service missing in agency: " + dao.getAllAgencies().iterator().next()); + private boolean isWeekend(Date date) { + Calendar cal = new GregorianCalendar(); + cal.setTime(date); + int day = cal.get(Calendar.DAY_OF_WEEK); + if (day == Calendar.SUNDAY || day == Calendar.SATURDAY) { + return true; } + return false; } private Date constructDate(ServiceDate date) { @@ -150,6 +150,18 @@ private Date removeTime(Date date) { return date; } + private Date addDays(Date date, int daysToAdd) { + Calendar cal = Calendar.getInstance(); + cal.setTime(date); + cal.add(Calendar.DATE, daysToAdd); + return cal.getTime(); + } + + private String formatDate(Date date) { + SimpleDateFormat dateFormat = new SimpleDateFormat("EEE MMM dd yyyy"); + return dateFormat.format(date); + } + private ServiceDate createServiceDate(Date date) { Calendar calendar = Calendar.getInstance(); calendar.setTime(date);