From b1100bfeeb0c13915cac329be6de5c4c8e9ad109 Mon Sep 17 00:00:00 2001 From: scrudden Date: Sat, 18 Aug 2018 11:52:20 +0100 Subject: [PATCH 01/24] Adding Kalman predictions for frequency based services. Moved alot of directories to seperate scheduled implementation from frequency implementation. This is a WIP. --- .../org/transitclock/applications/Core.java | 2 +- .../ArrivalDepartureGeneratorDefaultImpl.java | 6 +- .../core/PredictionGenerator.java | 12 +- .../transitclock/core/TravelTimeDetails.java | 2 +- .../core/dataCache/ErrorCacheFactory.java | 2 +- .../TripDataHistoryCacheFactory.java | 2 +- .../transitclock/core/dataCache/TripKey.java | 6 + .../dataCache/ehcache/KalmanErrorCache.java | 2 +- .../{ => frequency}/TripDataHistoryCache.java | 4 +- .../scheduled/TripDataHistoryCache.java | 334 ++++++++++++++++++ .../FrequencyBasedHistoricalAverageCache.java | 2 +- .../dataCache/jcs/DwellTimeModelCache.java | 25 +- .../jcs/frequency/TripDataHistoryCache.java | 182 ++++++++++ .../{ => scheduled}/TripDataHistoryCache.java | 2 +- .../ScheduleBasedHistoricalAverageCache.java | 2 +- .../HoldingTimeGeneratorDefaultImpl.java | 52 +++ .../kalman/KalmanPrediction.java | 1 - .../kalman/Prediction.java | 5 - .../KalmanPredictionGeneratorImpl.java | 240 +++++++++++++ .../KalmanPredictionGeneratorImpl.java | 9 +- .../LastVehiclePredictionGeneratorImpl.java | 2 +- .../DwellTimePredictionGeneratorImpl.java | 81 +++++ .../DwellTimePredictionGeneratorImpl.java | 4 +- .../{ => scheduled}/TransitClockRLS.java | 2 +- .../bullrunner/BullrunnerPlaybackModule.java | 186 ++++++++++ 25 files changed, 1129 insertions(+), 38 deletions(-) rename transitclock/src/main/java/org/transitclock/core/dataCache/ehcache/{ => frequency}/TripDataHistoryCache.java (95%) create mode 100755 transitclock/src/main/java/org/transitclock/core/dataCache/ehcache/scheduled/TripDataHistoryCache.java create mode 100644 transitclock/src/main/java/org/transitclock/core/dataCache/jcs/frequency/TripDataHistoryCache.java rename transitclock/src/main/java/org/transitclock/core/dataCache/jcs/{ => scheduled}/TripDataHistoryCache.java (99%) delete mode 100755 transitclock/src/main/java/org/transitclock/core/predictiongenerator/kalman/Prediction.java create mode 100755 transitclock/src/main/java/org/transitclock/core/predictiongenerator/kalman/frequency/KalmanPredictionGeneratorImpl.java rename transitclock/src/main/java/org/transitclock/core/predictiongenerator/kalman/{ => scheduled}/KalmanPredictionGeneratorImpl.java (93%) create mode 100644 transitclock/src/main/java/org/transitclock/core/predictiongenerator/rls/dwell/frequency/DwellTimePredictionGeneratorImpl.java rename transitclock/src/main/java/org/transitclock/core/predictiongenerator/rls/dwell/{ => scheduled}/DwellTimePredictionGeneratorImpl.java (94%) rename transitclock/src/main/java/org/transitclock/core/predictiongenerator/rls/dwell/{ => scheduled}/TransitClockRLS.java (94%) create mode 100644 transitclock/src/main/java/org/transitclock/custom/bullrunner/BullrunnerPlaybackModule.java diff --git a/transitclock/src/main/java/org/transitclock/applications/Core.java b/transitclock/src/main/java/org/transitclock/applications/Core.java index 75a0c173f..02bb4f7a3 100755 --- a/transitclock/src/main/java/org/transitclock/applications/Core.java +++ b/transitclock/src/main/java/org/transitclock/applications/Core.java @@ -44,7 +44,7 @@ import org.transitclock.core.dataCache.TripDataHistoryCacheFactory; import org.transitclock.core.dataCache.VehicleDataCache; import org.transitclock.core.dataCache.ehcache.StopArrivalDepartureCache; -import org.transitclock.core.dataCache.ehcache.TripDataHistoryCache; +import org.transitclock.core.dataCache.ehcache.scheduled.TripDataHistoryCache; import org.transitclock.core.dataCache.frequency.FrequencyBasedHistoricalAverageCache; import org.transitclock.core.dataCache.scheduled.ScheduleBasedHistoricalAverageCache; import org.transitclock.db.hibernate.DataDbLogger; diff --git a/transitclock/src/main/java/org/transitclock/core/ArrivalDepartureGeneratorDefaultImpl.java b/transitclock/src/main/java/org/transitclock/core/ArrivalDepartureGeneratorDefaultImpl.java index 6391fcdc6..34e412c24 100755 --- a/transitclock/src/main/java/org/transitclock/core/ArrivalDepartureGeneratorDefaultImpl.java +++ b/transitclock/src/main/java/org/transitclock/core/ArrivalDepartureGeneratorDefaultImpl.java @@ -33,9 +33,10 @@ import org.transitclock.core.dataCache.TripDataHistoryCacheFactory; import org.transitclock.core.dataCache.VehicleStateManager; import org.transitclock.core.dataCache.ehcache.StopArrivalDepartureCache; -import org.transitclock.core.dataCache.ehcache.TripDataHistoryCache; +import org.transitclock.core.dataCache.ehcache.scheduled.TripDataHistoryCache; import org.transitclock.core.dataCache.frequency.FrequencyBasedHistoricalAverageCache; import org.transitclock.core.dataCache.scheduled.ScheduleBasedHistoricalAverageCache; +import org.transitclock.core.holdingmethod.HoldingTimeGeneratorDefaultImpl; import org.transitclock.core.holdingmethod.HoldingTimeGeneratorFactory; import org.transitclock.core.predAccuracy.PredictionAccuracyModule; @@ -377,6 +378,9 @@ private void updateCache(VehicleState vehicleState, ArrivalDeparture arrivalDepa HoldingTimeGeneratorFactory.getInstance().handleDeparture(vehicleState, arrivalDeparture); } + if(HoldingTimeGeneratorDefaultImpl.getOrderedListOfVehicles("66")!=null) + logger.info("ORDER:"+HoldingTimeGeneratorDefaultImpl.getOrderedListOfVehicles("66").toString()); + /* if(HoldingTimeGeneratorFactory.getInstance()!=null) { diff --git a/transitclock/src/main/java/org/transitclock/core/PredictionGenerator.java b/transitclock/src/main/java/org/transitclock/core/PredictionGenerator.java index 2d8870f92..a37898451 100644 --- a/transitclock/src/main/java/org/transitclock/core/PredictionGenerator.java +++ b/transitclock/src/main/java/org/transitclock/core/PredictionGenerator.java @@ -38,7 +38,7 @@ import org.transitclock.core.dataCache.TripDataHistoryCacheInterface; import org.transitclock.core.dataCache.TripKey; import org.transitclock.core.dataCache.ehcache.StopArrivalDepartureCache; -import org.transitclock.core.dataCache.ehcache.TripDataHistoryCache; +import org.transitclock.core.dataCache.ehcache.scheduled.TripDataHistoryCache; import org.transitclock.db.structs.ArrivalDeparture; import org.transitclock.db.structs.AvlReport; import org.transitclock.db.structs.Block; @@ -283,10 +283,12 @@ protected List lastDaysTimes(TripDataHistoryCacheInterface ca if (arrival != null && departure != null) { TravelTimeDetails travelTimeDetails=new TravelTimeDetails(departure, arrival); - - times.add(travelTimeDetails); - - num_found++; + + if(travelTimeDetails.getTravelTime()!=-1) + { + times.add(travelTimeDetails); + num_found++; + } } } } diff --git a/transitclock/src/main/java/org/transitclock/core/TravelTimeDetails.java b/transitclock/src/main/java/org/transitclock/core/TravelTimeDetails.java index 6cd03be97..c22760f93 100644 --- a/transitclock/src/main/java/org/transitclock/core/TravelTimeDetails.java +++ b/transitclock/src/main/java/org/transitclock/core/TravelTimeDetails.java @@ -53,7 +53,7 @@ public boolean sanityCheck() { long travelTime=this.arrival.getTime()-this.getDeparture().getTime(); - if(travelTime<0||travelTime>maxTravelTime.getValue()) + if(travelTime<=0||travelTime>maxTravelTime.getValue()) { return false; }else diff --git a/transitclock/src/main/java/org/transitclock/core/dataCache/ErrorCacheFactory.java b/transitclock/src/main/java/org/transitclock/core/dataCache/ErrorCacheFactory.java index 6939fb24f..4d1e4b033 100644 --- a/transitclock/src/main/java/org/transitclock/core/dataCache/ErrorCacheFactory.java +++ b/transitclock/src/main/java/org/transitclock/core/dataCache/ErrorCacheFactory.java @@ -10,7 +10,7 @@ public class ErrorCacheFactory { private static StringConfigValue className = new StringConfigValue("transitclock.core.cache.errorCacheClass", - "org.transitclock.core.dataCache.jcs.KalmanErrorCache", + "org.transitclock.core.dataCache.ehcache.KalmanErrorCache", "Specifies the class used to cache the Kalamn error values."); private static ErrorCache singleton = null; diff --git a/transitclock/src/main/java/org/transitclock/core/dataCache/TripDataHistoryCacheFactory.java b/transitclock/src/main/java/org/transitclock/core/dataCache/TripDataHistoryCacheFactory.java index 2f1d6fa37..00548236f 100644 --- a/transitclock/src/main/java/org/transitclock/core/dataCache/TripDataHistoryCacheFactory.java +++ b/transitclock/src/main/java/org/transitclock/core/dataCache/TripDataHistoryCacheFactory.java @@ -10,7 +10,7 @@ public class TripDataHistoryCacheFactory { private static StringConfigValue className = new StringConfigValue("transitclock.core.cache.tripDataHistoryCache", - "org.transitclock.core.dataCache.jcs.TripDataHistoryCache", + "org.transitclock.core.dataCache.ehcache.frequency.TripDataHistoryCache", "Specifies the class used to cache the arrival and departures for a trip."); private static TripDataHistoryCacheInterface singleton = null; diff --git a/transitclock/src/main/java/org/transitclock/core/dataCache/TripKey.java b/transitclock/src/main/java/org/transitclock/core/dataCache/TripKey.java index 1b821b26b..cfa1ff89b 100755 --- a/transitclock/src/main/java/org/transitclock/core/dataCache/TripKey.java +++ b/transitclock/src/main/java/org/transitclock/core/dataCache/TripKey.java @@ -86,6 +86,12 @@ public String toString() { return "TripKey [tripId=" + tripId + ", tripStartDate=" + tripStartDate + ", startTime=" + startTime + "]"; } + public void setStartTime(Integer time) { + // TODO Auto-generated method stub + this.startTime=time; + + } + diff --git a/transitclock/src/main/java/org/transitclock/core/dataCache/ehcache/KalmanErrorCache.java b/transitclock/src/main/java/org/transitclock/core/dataCache/ehcache/KalmanErrorCache.java index 8038e1cd0..03141d8cd 100644 --- a/transitclock/src/main/java/org/transitclock/core/dataCache/ehcache/KalmanErrorCache.java +++ b/transitclock/src/main/java/org/transitclock/core/dataCache/ehcache/KalmanErrorCache.java @@ -30,7 +30,7 @@ public class KalmanErrorCache implements ErrorCache { * @return */ - KalmanErrorCache() { + public KalmanErrorCache() { CacheManager cm = CacheManager.getInstance(); if (cm.getCache(cacheName) == null) { diff --git a/transitclock/src/main/java/org/transitclock/core/dataCache/ehcache/TripDataHistoryCache.java b/transitclock/src/main/java/org/transitclock/core/dataCache/ehcache/frequency/TripDataHistoryCache.java similarity index 95% rename from transitclock/src/main/java/org/transitclock/core/dataCache/ehcache/TripDataHistoryCache.java rename to transitclock/src/main/java/org/transitclock/core/dataCache/ehcache/frequency/TripDataHistoryCache.java index 0246f6e17..298513b44 100755 --- a/transitclock/src/main/java/org/transitclock/core/dataCache/ehcache/TripDataHistoryCache.java +++ b/transitclock/src/main/java/org/transitclock/core/dataCache/ehcache/frequency/TripDataHistoryCache.java @@ -1,7 +1,7 @@ /** * */ -package org.transitclock.core.dataCache.ehcache; +package org.transitclock.core.dataCache.ehcache.frequency; import java.util.Collections; import java.util.Comparator; @@ -78,7 +78,7 @@ public static TripDataHistoryCacheInterface getInstance() { return singleton; } - private TripDataHistoryCache() { + public TripDataHistoryCache() { CacheManager cm = CacheManager.getInstance(); EvictionAgePolicy evictionPolicy = null; if(tripDataCacheMaxAgeSec!=null) diff --git a/transitclock/src/main/java/org/transitclock/core/dataCache/ehcache/scheduled/TripDataHistoryCache.java b/transitclock/src/main/java/org/transitclock/core/dataCache/ehcache/scheduled/TripDataHistoryCache.java new file mode 100755 index 000000000..e771429d5 --- /dev/null +++ b/transitclock/src/main/java/org/transitclock/core/dataCache/ehcache/scheduled/TripDataHistoryCache.java @@ -0,0 +1,334 @@ +/** + * + */ +package org.transitclock.core.dataCache.ehcache.scheduled; + +import java.util.Collections; +import java.util.Comparator; +import java.util.Date; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.LinkedList; +import java.util.List; + +import net.sf.ehcache.Cache; +import net.sf.ehcache.CacheManager; +import net.sf.ehcache.Element; +import net.sf.ehcache.config.CacheConfiguration; +import net.sf.ehcache.store.Policy; + +import org.apache.commons.beanutils.BeanComparator; +import org.apache.commons.lang3.time.DateUtils; +import org.hibernate.Criteria; +import org.hibernate.criterion.Restrictions; +import org.hibernate.Session; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.transitclock.applications.Core; +import org.transitclock.config.IntegerConfigValue; +import org.transitclock.core.dataCache.ArrivalDepartureComparator; +import org.transitclock.core.dataCache.TripDataHistoryCacheFactory; +import org.transitclock.core.dataCache.TripDataHistoryCacheInterface; +import org.transitclock.core.dataCache.TripKey; +import org.transitclock.db.structs.ArrivalDeparture; +import org.transitclock.db.structs.Block; +import org.transitclock.db.structs.Trip; +import org.transitclock.gtfs.DbConfig; +import org.transitclock.gtfs.GtfsData; +import org.transitclock.utils.Time; + +/** + * @author Sean Og Crudden + * This is a Cache to hold historical arrival departure data for frequency based trips. It + * is intended to look up a trips historical data when a trip starts and + * place in cache for use in generating predictions based on a Kalman + * filter. + * + */ +public class TripDataHistoryCache implements TripDataHistoryCacheInterface{ + private static TripDataHistoryCacheInterface singleton = new TripDataHistoryCache(); + + private static boolean debug = false; + + final private static String cacheByTrip = "arrivalDeparturesByTrip"; + + + private static final Logger logger = LoggerFactory + .getLogger(TripDataHistoryCache.class); + + private Cache cache = null; + + /** + * Default is 4 as we need 3 days worth for Kalman Filter implementation + */ + private static final IntegerConfigValue tripDataCacheMaxAgeSec = new IntegerConfigValue( + "transitclock.tripdatacache.tripDataCacheMaxAgeSec", + 15 * Time.SEC_PER_DAY, + "How old an arrivaldeparture has to be before it is removed from the cache "); + + /** + * Gets the singleton instance of this class. + * + * @return + */ + public static TripDataHistoryCacheInterface getInstance() { + return singleton; + } + + private TripDataHistoryCache() { + CacheManager cm = CacheManager.getInstance(); + EvictionAgePolicy evictionPolicy = null; + if(tripDataCacheMaxAgeSec!=null) + { + evictionPolicy = new EvictionAgePolicy( + tripDataCacheMaxAgeSec.getValue() * Time.MS_PER_SEC); + }else + { + evictionPolicy = new EvictionAgePolicy( + 15 * Time.SEC_PER_DAY *Time.MS_PER_SEC); + } + + if (cm.getCache(cacheByTrip) == null) { + cm.addCache(cacheByTrip); + } + cache = cm.getCache(cacheByTrip); + + //CacheConfiguration config = cache.getCacheConfiguration(); + /*TODO We need to refine the eviction policy. */ + cache.setMemoryStoreEvictionPolicy(evictionPolicy); + } + /* (non-Javadoc) + * @see org.transitclock.core.dataCache.TripDataHistoryCacheInterface#getKeys() + */ + @Override + public List getKeys() + { + return cache.getKeys(); + } + /* (non-Javadoc) + * @see org.transitclock.core.dataCache.TripDataHistoryCacheInterface#logCache(org.slf4j.Logger) + */ + @Override + public void logCache(Logger logger) + { + logger.debug("Cache content log."); + @SuppressWarnings("unchecked") + List keys = cache.getKeys(); + + for(TripKey key : keys) + { + Element result=cache.get(key); + if(result!=null) + { + logger.debug("Key: "+key.toString()); + @SuppressWarnings("unchecked") + + List ads=(List) result.getObjectValue(); + + for(ArrivalDeparture ad : ads) + { + logger.debug(ad.toString()); + } + } + } + + } + + /* (non-Javadoc) + * @see org.transitclock.core.dataCache.TripDataHistoryCacheInterface#getTripHistory(org.transitclock.core.dataCache.TripKey) + */ + @Override + @SuppressWarnings("unchecked") + public List getTripHistory(TripKey tripKey) { + + //logger.debug(cache.toString()); + + Element result = cache.get(tripKey); + + if(result!=null) + { + return (List) result.getObjectValue(); + } + else + { + return null; + } + } + + /* (non-Javadoc) + * @see org.transitclock.core.dataCache.TripDataHistoryCacheInterface#putArrivalDeparture(org.transitclock.db.structs.ArrivalDeparture) + */ + @Override + @SuppressWarnings("unchecked") + synchronized public TripKey putArrivalDeparture(ArrivalDeparture arrivalDeparture) { + + logger.debug("Putting :"+arrivalDeparture.toString() + " in TripDataHistoryCache cache."); + /* just put todays time in for last three days to aid development. This means it will kick in in 1 days rather than 3. Perhaps be a good way to start rather than using default transiTime method but I doubt it. */ + int days_back=1; + if(debug) + days_back=3; + TripKey tripKey=null; + + for(int i=0;i < days_back;i++) + { + Date nearestDay = DateUtils.truncate(new Date(arrivalDeparture.getTime()), Calendar.DAY_OF_MONTH); + + nearestDay=DateUtils.addDays(nearestDay, i*-1); + + DbConfig dbConfig = Core.getInstance().getDbConfig(); + + Trip trip=dbConfig.getTrip(arrivalDeparture.getTripId()); + + if(trip!=null) + { + + tripKey = new TripKey(arrivalDeparture.getTripId(), + nearestDay, + trip.getStartTime()); + + List list = null; + + Element result = cache.get(tripKey); + + if (result != null && result.getObjectValue() != null) { + list = (List) result.getObjectValue(); + cache.remove(tripKey); + } else { + list = new ArrayList(); + } + + list.add(arrivalDeparture); + + Element arrivalDepartures = new Element(tripKey, Collections.synchronizedList(list)); + + cache.put(arrivalDepartures); + } + + + } + return tripKey; + } + + /* (non-Javadoc) + * @see org.transitclock.core.dataCache.TripDataHistoryCacheInterface#populateCacheFromDb(org.hibernate.Session, java.util.Date, java.util.Date) + */ + + @Override + public void populateCacheFromDb(Session session, Date startDate, Date endDate) + { + Criteria criteria =session.createCriteria(ArrivalDeparture.class); + + @SuppressWarnings("unchecked") + List results=criteria.add(Restrictions.between("time", startDate, endDate)).list(); + + for(ArrivalDeparture result : results) + { + // TODO this might be better done in the database. + if(GtfsData.routeNotFiltered(result.getRouteId())) + { + TripDataHistoryCacheFactory.getInstance().putArrivalDeparture(result); + } + } + } + + /* (non-Javadoc) + * @see org.transitclock.core.dataCache.ehcache.test#findPreviousArrivalEvent(java.util.List, org.transitclock.db.structs.ArrivalDeparture) + */ + @Override + public ArrivalDeparture findPreviousArrivalEvent(List arrivalDepartures,ArrivalDeparture current) + { + Collections.sort(arrivalDepartures, new ArrivalDepartureComparator()); + for (ArrivalDeparture tocheck : emptyIfNull(arrivalDepartures)) + { + if(tocheck.getStopId().equals(current.getStopId()) && (current.isDeparture() && tocheck.isArrival())) + { + return tocheck; + } + } + return null; + } + /* (non-Javadoc) + * @see org.transitclock.core.dataCache.ehcache.test#findPreviousDepartureEvent(java.util.List, org.transitclock.db.structs.ArrivalDeparture) + */ + @Override + public ArrivalDeparture findPreviousDepartureEvent(List arrivalDepartures,ArrivalDeparture current) + { + Collections.sort(arrivalDepartures, new ArrivalDepartureComparator()); + for (ArrivalDeparture tocheck : emptyIfNull(arrivalDepartures)) + { + try { + if(tocheck.getStopPathIndex()==(current.getStopPathIndex()-1) && (current.isArrival() && tocheck.isDeparture())) + { + return tocheck; + } + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + return null; + } + + private static Iterable emptyIfNull(Iterable iterable) { + return iterable == null ? Collections. emptyList() : iterable; + } + /** + * This policy evicts arrival departures from the cache + * when they are X (age) number of milliseconds old + * + */ + private class EvictionAgePolicy implements Policy { + private String name = "AGE"; + + private long age = 0L; + + public EvictionAgePolicy(long age) { + super(); + this.age = age; + } + + @Override + public boolean compare(Element arg0, Element arg1) { + if (arg0.getObjectKey() instanceof TripKey + && arg1.getObjectKey() instanceof TripKey) { + if (((TripKey) arg0.getObjectKey()).getTripStartDate().after( + ((TripKey) arg1.getObjectKey()).getTripStartDate())) { + return true; + } + if (((TripKey) arg0.getObjectKey()).getTripStartDate() + .compareTo( + ((TripKey) arg1.getObjectKey()) + .getTripStartDate()) == 0) { + if (((TripKey) arg0.getObjectKey()).getStartTime() > ((TripKey) arg1 + .getObjectKey()).getStartTime()) { + return true; + } + } + } + return false; + } + + @Override + public String getName() { + return name; + } + + @Override + public Element selectedBasedOnPolicy(Element[] arg0, Element arg1) { + + for (int i = 0; i < arg0.length; i++) { + + if (arg0[i].getObjectKey() instanceof TripKey) { + TripKey key = (TripKey) arg0[i].getObjectKey(); + + if (Calendar.getInstance().getTimeInMillis() + - key.getTripStartDate().getTime() + + (key.getStartTime().intValue() * 1000) > age) { + return arg0[i]; + } + } + } + return null; + } + } +} diff --git a/transitclock/src/main/java/org/transitclock/core/dataCache/frequency/FrequencyBasedHistoricalAverageCache.java b/transitclock/src/main/java/org/transitclock/core/dataCache/frequency/FrequencyBasedHistoricalAverageCache.java index f8433143d..a6cebcf4b 100755 --- a/transitclock/src/main/java/org/transitclock/core/dataCache/frequency/FrequencyBasedHistoricalAverageCache.java +++ b/transitclock/src/main/java/org/transitclock/core/dataCache/frequency/FrequencyBasedHistoricalAverageCache.java @@ -27,7 +27,7 @@ import org.transitclock.core.Indices; import org.transitclock.core.VehicleState; import org.transitclock.core.dataCache.*; -import org.transitclock.core.dataCache.ehcache.TripDataHistoryCache; +import org.transitclock.core.dataCache.ehcache.scheduled.TripDataHistoryCache; import org.transitclock.db.structs.ArrivalDeparture; import org.transitclock.db.structs.Trip; import org.transitclock.gtfs.DbConfig; diff --git a/transitclock/src/main/java/org/transitclock/core/dataCache/jcs/DwellTimeModelCache.java b/transitclock/src/main/java/org/transitclock/core/dataCache/jcs/DwellTimeModelCache.java index fe5b07838..84a9532fc 100644 --- a/transitclock/src/main/java/org/transitclock/core/dataCache/jcs/DwellTimeModelCache.java +++ b/transitclock/src/main/java/org/transitclock/core/dataCache/jcs/DwellTimeModelCache.java @@ -18,8 +18,8 @@ import org.transitclock.core.dataCache.DwellTimeCacheKey; import org.transitclock.core.dataCache.StopArrivalDepartureCacheFactory; import org.transitclock.core.dataCache.StopArrivalDepartureCacheKey; -import org.transitclock.core.predictiongenerator.rls.dwell.DwellTimePredictionGeneratorImpl; -import org.transitclock.core.predictiongenerator.rls.dwell.TransitClockRLS; +import org.transitclock.core.predictiongenerator.rls.dwell.scheduled.DwellTimePredictionGeneratorImpl; +import org.transitclock.core.predictiongenerator.rls.dwell.scheduled.TransitClockRLS; import org.transitclock.db.structs.ArrivalDeparture; import org.transitclock.db.structs.Block; import org.transitclock.db.structs.Headway; @@ -32,10 +32,10 @@ public class DwellTimeModelCache implements org.transitclock.core.dataCache.Dwel final private static String cacheName = "DwellTimeModelCache"; - private static IntegerConfigValue maxDwellTimeAllowedInModel = new IntegerConfigValue("org.transitclock.core.dataCache.jcs.maxDwellTimeAllowedInModel", 120000, "Max dwell time to be considered in dwell RLS algotithm."); - private static LongConfigValue maxHeadwayAllowedInModel = new LongConfigValue("org.transitclock.core.dataCache.jcs.maxHeadwayAllowedInModel", 1*Time.MS_PER_HOUR, "Max dwell time to be considered in dwell RLS algotithm."); + private static IntegerConfigValue maxDwellTimeAllowedInModel = new IntegerConfigValue("org.transitclock.core.dataCache.jcs.maxDwellTimeAllowedInModel", 2 * Time.MS_PER_MIN, "Max dwell time to be considered in dwell RLS algotithm."); + private static LongConfigValue maxHeadwayAllowedInModel = new LongConfigValue("org.transitclock.core.dataCache.jcs.maxHeadwayAllowedInModel", 1*Time.MS_PER_HOUR, "Max headway to be considered in dwell RLS algotithm."); - private static DoubleConfigValue lambda = new DoubleConfigValue("org.transitclock.core.dataCache.jcs.lambda", 0.5, "This sets the rate at which the RLS algorithm forgets old values. Value are between 0 and 1. With 0 being the most forgetful."); + private static DoubleConfigValue lambda = new DoubleConfigValue("org.transitclock.core.dataCache.jcs.lambda", 0.75, "This sets the rate at which the RLS algorithm forgets old values. Value are between 0 and 1. With 0 being the most forgetful."); private CacheAccess cache = null; @@ -119,11 +119,16 @@ public void addSample(ArrivalDeparture departure) { long dwelltime=departure.getTime()-arrival.getTime(); headway.setTripId(arrival.getTripId()); - /* Leave out silly values as they are most likely errors or unusual circumstance. */ - if(dwelltime=0) + { + /* Leave out silly values as they are most likely errors or unusual circumstance. */ + if(dwelltime> cache = null; + + @Override + public List getKeys() { + ArrayList fulllist=new ArrayList(); + Set names = JCS.getGroupCacheInstance(cacheName).getGroupNames(); + + for(String name:names) + { + Set keys = JCS.getGroupCacheInstance(cacheName).getGroupKeys(name); + + for(Object key:keys) + { + fulllist.add((TripKey)key); + } + } + return fulllist; + } + + public TripDataHistoryCache() { + cache = JCS.getInstance(cacheName); + } + + @Override + public void logCache(Logger logger) { + + logger.debug("Cache content log. Not implemented."); + } + + @Override + public List getTripHistory(TripKey tripKey) { + + /* this is what gets the trip from the buckets */ + int time = FrequencyBasedHistoricalAverageCache.round(tripKey.getStartTime(), FrequencyBasedHistoricalAverageCache.getCacheIncrementsForFrequencyService()); + + tripKey.setStartTime(time); + + return cache.get(tripKey); + } + + @Override + synchronized public TripKey putArrivalDeparture(ArrivalDeparture arrivalDeparture) { + logger.debug("Putting :"+arrivalDeparture.toString() + " in TripDataHistoryCache cache."); + /* just put todays time in for last three days to aid development. This means it will kick in in 1 days rather than 3. Perhaps be a good way to start rather than using default transiTime method but I doubt it. */ + int days_back=1; + + TripKey tripKey=null; + + for(int i=0;i < days_back;i++) + { + Date nearestDay = DateUtils.truncate(new Date(arrivalDeparture.getTime()), Calendar.DAY_OF_MONTH); + + nearestDay=DateUtils.addDays(nearestDay, i*-1); + + DbConfig dbConfig = Core.getInstance().getDbConfig(); + + Trip trip=dbConfig.getTrip(arrivalDeparture.getTripId()); + + if(trip!=null) + { + Integer time=FrequencyBasedHistoricalAverageCache.secondsFromMidnight(arrivalDeparture.getDate(),2); + + /* this is what gets the trip from the buckets */ + time=FrequencyBasedHistoricalAverageCache.round(time, FrequencyBasedHistoricalAverageCache.getCacheIncrementsForFrequencyService()); + + tripKey = new TripKey(arrivalDeparture.getTripId(), + nearestDay, + time); + + List list = cache.get(tripKey); + + if(list==null) + list = new ArrayList(); + + list.add(arrivalDeparture); + cache.put(tripKey, Collections.synchronizedList(list)); + } + } + return tripKey; + + } + + /* (non-Javadoc) + * @see org.transitclock.core.dataCache.TripDataHistoryCacheInterface#populateCacheFromDb(org.hibernate.Session, java.util.Date, java.util.Date) + */ + + @Override + public void populateCacheFromDb(Session session, Date startDate, Date endDate) + { + Criteria criteria =session.createCriteria(ArrivalDeparture.class); + + @SuppressWarnings("unchecked") + List results=criteria.add(Restrictions.between("time", startDate, endDate)).list(); + + for(ArrivalDeparture result : results) + { + // TODO this might be better done in the database. + if(GtfsData.routeNotFiltered(result.getRouteId())) + { + TripDataHistoryCacheFactory.getInstance().putArrivalDeparture(result); + } + } + } + + /* (non-Javadoc) + * @see org.transitclock.core.dataCache.ehcache.test#findPreviousArrivalEvent(java.util.List, org.transitclock.db.structs.ArrivalDeparture) + */ + @Override + public ArrivalDeparture findPreviousArrivalEvent(List arrivalDepartures,ArrivalDeparture current) + { + Collections.sort(arrivalDepartures, new ArrivalDepartureComparator()); + for (ArrivalDeparture tocheck : emptyIfNull(arrivalDepartures)) + { + if(tocheck.getStopId().equals(current.getStopId()) && (current.isDeparture() && tocheck.isArrival())) + { + return tocheck; + } + } + return null; + } + /* (non-Javadoc) + * @see org.transitclock.core.dataCache.ehcache.test#findPreviousDepartureEvent(java.util.List, org.transitclock.db.structs.ArrivalDeparture) + */ + @Override + public ArrivalDeparture findPreviousDepartureEvent(List arrivalDepartures,ArrivalDeparture current) + { + Collections.sort(arrivalDepartures, new ArrivalDepartureComparator()); + for (ArrivalDeparture tocheck : emptyIfNull(arrivalDepartures)) + { + try { + if(tocheck.getStopPathIndex()==(current.getStopPathIndex()-1) && (current.isArrival() && tocheck.isDeparture())) + { + return tocheck; + } + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + return null; + } + private static Iterable emptyIfNull(Iterable iterable) { + return iterable == null ? Collections. emptyList() : iterable; + } +} diff --git a/transitclock/src/main/java/org/transitclock/core/dataCache/jcs/TripDataHistoryCache.java b/transitclock/src/main/java/org/transitclock/core/dataCache/jcs/scheduled/TripDataHistoryCache.java similarity index 99% rename from transitclock/src/main/java/org/transitclock/core/dataCache/jcs/TripDataHistoryCache.java rename to transitclock/src/main/java/org/transitclock/core/dataCache/jcs/scheduled/TripDataHistoryCache.java index c37eb222a..6ddb03a08 100644 --- a/transitclock/src/main/java/org/transitclock/core/dataCache/jcs/TripDataHistoryCache.java +++ b/transitclock/src/main/java/org/transitclock/core/dataCache/jcs/scheduled/TripDataHistoryCache.java @@ -1,4 +1,4 @@ -package org.transitclock.core.dataCache.jcs; +package org.transitclock.core.dataCache.jcs.scheduled; import java.util.ArrayList; import java.util.Calendar; diff --git a/transitclock/src/main/java/org/transitclock/core/dataCache/scheduled/ScheduleBasedHistoricalAverageCache.java b/transitclock/src/main/java/org/transitclock/core/dataCache/scheduled/ScheduleBasedHistoricalAverageCache.java index 4860ed664..9eb90d5d5 100644 --- a/transitclock/src/main/java/org/transitclock/core/dataCache/scheduled/ScheduleBasedHistoricalAverageCache.java +++ b/transitclock/src/main/java/org/transitclock/core/dataCache/scheduled/ScheduleBasedHistoricalAverageCache.java @@ -23,7 +23,7 @@ import org.transitclock.core.dataCache.StopPathCacheKey; import org.transitclock.core.dataCache.TripDataHistoryCacheFactory; import org.transitclock.core.dataCache.TripKey; -import org.transitclock.core.dataCache.ehcache.TripDataHistoryCache; +import org.transitclock.core.dataCache.ehcache.scheduled.TripDataHistoryCache; import org.transitclock.core.dataCache.frequency.FrequencyBasedHistoricalAverageCache; import org.transitclock.db.structs.ArrivalDeparture; import org.transitclock.db.structs.Trip; diff --git a/transitclock/src/main/java/org/transitclock/core/holdingmethod/HoldingTimeGeneratorDefaultImpl.java b/transitclock/src/main/java/org/transitclock/core/holdingmethod/HoldingTimeGeneratorDefaultImpl.java index 217f4e822..1df6c66f9 100755 --- a/transitclock/src/main/java/org/transitclock/core/holdingmethod/HoldingTimeGeneratorDefaultImpl.java +++ b/transitclock/src/main/java/org/transitclock/core/holdingmethod/HoldingTimeGeneratorDefaultImpl.java @@ -225,6 +225,58 @@ public HoldingTime generateHoldingTime(VehicleState vehicleState, ArrivalDepartu // Return null so has no effect. return null; } + public static List getOrderedListOfVehicles(String routeId) + { + int count=0; + boolean canorder=true; + List unordered=new ArrayList(); + List ordered=null; + for(VehicleState currentVehicleState:VehicleStateManager.getInstance().getVehiclesState()) + { + if(currentVehicleState.getTrip()!=null&¤tVehicleState.getTrip().getRoute().getId().equals(routeId)&¤tVehicleState.isPredictable()) + { + count++; + unordered.add(currentVehicleState); + if(currentVehicleState.getHeadway()==null) + { + canorder=false; + + } + } + } + if(canorder) + { + ordered=new ArrayList(); + + while(((count+1) > 0)&&unordered.size() > 0) + { + if(ordered.size()==0) + { + String first=unordered.get(0).getVehicleId(); + String second=unordered.get(0).getHeadway().getOtherVehicleId(); + + ordered.add(first); + count--; + ordered.add(second); + count--; + }else + { + ordered.add(VehicleStateManager.getInstance().getVehicleState(ordered.get(ordered.size()-1)).getHeadway().getOtherVehicleId()); + count--; + } + + } + // check first vehicle equals last vehicle + if(ordered.size()>1) + { + if(!ordered.get(ordered.size()-1).equals(ordered.get(0))) + { + return null; + } + } + } + return ordered; + } protected ArrayList getCurrentHoldingTimesForStop(String stopId) { ArrayList currentHoldingTimes=new ArrayList(); diff --git a/transitclock/src/main/java/org/transitclock/core/predictiongenerator/kalman/KalmanPrediction.java b/transitclock/src/main/java/org/transitclock/core/predictiongenerator/kalman/KalmanPrediction.java index 9ecdcd92c..a19712690 100755 --- a/transitclock/src/main/java/org/transitclock/core/predictiongenerator/kalman/KalmanPrediction.java +++ b/transitclock/src/main/java/org/transitclock/core/predictiongenerator/kalman/KalmanPrediction.java @@ -1,6 +1,5 @@ package org.transitclock.core.predictiongenerator.kalman; - /** * @author Sean Óg Crudden * diff --git a/transitclock/src/main/java/org/transitclock/core/predictiongenerator/kalman/Prediction.java b/transitclock/src/main/java/org/transitclock/core/predictiongenerator/kalman/Prediction.java deleted file mode 100755 index 81166d797..000000000 --- a/transitclock/src/main/java/org/transitclock/core/predictiongenerator/kalman/Prediction.java +++ /dev/null @@ -1,5 +0,0 @@ -package org.transitclock.core.predictiongenerator.kalman; - -public interface Prediction { - -} diff --git a/transitclock/src/main/java/org/transitclock/core/predictiongenerator/kalman/frequency/KalmanPredictionGeneratorImpl.java b/transitclock/src/main/java/org/transitclock/core/predictiongenerator/kalman/frequency/KalmanPredictionGeneratorImpl.java new file mode 100755 index 000000000..4744916fd --- /dev/null +++ b/transitclock/src/main/java/org/transitclock/core/predictiongenerator/kalman/frequency/KalmanPredictionGeneratorImpl.java @@ -0,0 +1,240 @@ +package org.transitclock.core.predictiongenerator.kalman.frequency; + +import java.util.Calendar; +import java.util.Date; +import java.util.List; + +import org.apache.commons.lang3.time.DateUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.transitclock.applications.Core; +import org.transitclock.config.BooleanConfigValue; +import org.transitclock.config.DoubleConfigValue; +import org.transitclock.config.IntegerConfigValue; +import org.transitclock.core.Indices; +import org.transitclock.core.SpatialMatch; +import org.transitclock.core.TravelTimeDetails; +import org.transitclock.core.VehicleState; +import org.transitclock.core.dataCache.ErrorCache; +import org.transitclock.core.dataCache.ErrorCacheFactory; +import org.transitclock.core.dataCache.KalmanErrorCacheKey; +import org.transitclock.core.dataCache.StopPathPredictionCache; +import org.transitclock.core.dataCache.TripDataHistoryCacheFactory; +import org.transitclock.core.dataCache.TripDataHistoryCacheInterface; +import org.transitclock.core.dataCache.VehicleStateManager; +import org.transitclock.core.dataCache.frequency.FrequencyBasedHistoricalAverageCache; +import org.transitclock.core.predictiongenerator.PredictionComponentElementsGenerator; +import org.transitclock.core.predictiongenerator.average.frequency.HistoricalAveragePredictionGeneratorImpl; +import org.transitclock.core.predictiongenerator.kalman.KalmanPrediction; +import org.transitclock.core.predictiongenerator.kalman.KalmanPredictionResult; +import org.transitclock.core.predictiongenerator.kalman.TripSegment; +import org.transitclock.core.predictiongenerator.kalman.Vehicle; +import org.transitclock.core.predictiongenerator.kalman.VehicleStopDetail; +import org.transitclock.db.structs.AvlReport; +import org.transitclock.db.structs.PredictionForStopPath; + +/** + * @author Sean Óg Crudden This is a prediction generator that uses a Kalman + * filter to provide predictions for a frequency based service. + */ +public class KalmanPredictionGeneratorImpl extends HistoricalAveragePredictionGeneratorImpl + implements PredictionComponentElementsGenerator { + + private String alternative="LastVehiclePredictionGeneratorImpl"; + + /* + * TODO I think this needs to be a minimum of three and if just two will use + * historical value. + */ + private static final IntegerConfigValue minKalmanDays = new IntegerConfigValue( + "transitclock.prediction.data.kalman.mindays", new Integer(3), + "Min number of days trip data that needs to be available before Kalman prediciton is used instead of default transiTime prediction."); + + private static final IntegerConfigValue maxKalmanDays = new IntegerConfigValue( + "transitclock.prediction.data.kalman.maxdays", new Integer(3), + "Max number of historical days trips to include in Kalman prediction calculation."); + + private static final IntegerConfigValue maxKalmanDaysToSearch = new IntegerConfigValue( + "transitclock.prediction.data.kalman.maxdaystoseach", new Integer(21), + "Max number of days to look back for data. This will also be effected by how old the data in the cache is."); + + private static final DoubleConfigValue initialErrorValue = new DoubleConfigValue( + "transitclock.prediction.data.kalman.initialerrorvalue", new Double(100), + "Initial Kalman error value to use to start filter."); + + /* May be better to use the default implementation as it splits things down into segments. */ + private static final BooleanConfigValue useKalmanForPartialStopPaths = new BooleanConfigValue ( + "transitclock.prediction.data.kalman.usekalmanforpartialstoppaths", new Boolean(true), + "Will use Kalman prediction to get to first stop of prediction." + ); + + + private static final Logger logger = LoggerFactory.getLogger(KalmanPredictionGeneratorImpl.class); + + /* + * (non-Javadoc) + * + * @see + * org.transitclock.core.PredictionGeneratorDefaultImpl#getTravelTimeForPath + * (org.transitclock.core.Indices, org.transitclock.db.structs.AvlReport) + */ + @Override + public long getTravelTimeForPath(Indices indices, AvlReport avlReport, VehicleState vehicleState) { + + logger.debug("Calling frequency based Kalman prediction algorithm for : "+indices.toString()); + + + Integer time=FrequencyBasedHistoricalAverageCache.secondsFromMidnight(avlReport.getDate(),2); + + time=FrequencyBasedHistoricalAverageCache.round(time, FrequencyBasedHistoricalAverageCache.getCacheIncrementsForFrequencyService()); + + TripDataHistoryCacheInterface tripCache = TripDataHistoryCacheFactory.getInstance(); + + ErrorCache kalmanErrorCache = ErrorCacheFactory.getInstance(); + + VehicleStateManager vehicleStateManager = VehicleStateManager.getInstance(); + + VehicleState currentVehicleState = vehicleStateManager.getVehicleState(avlReport.getVehicleId()); + + TravelTimeDetails travelTimeDetails = this.getLastVehicleTravelTime(currentVehicleState, indices); + + + /* + * The first vehicle of the day should use schedule or historic data to + * make prediction. Cannot use Kalman as yesterdays vehicle will have + * little to say about todays. + */ + if (travelTimeDetails!=null) { + + logger.debug("Kalman has last vehicle info for : " +indices.toString()+ " : "+travelTimeDetails); + + Date nearestDay = DateUtils.truncate(avlReport.getDate(), Calendar.DAY_OF_MONTH); + + List lastDaysTimes = lastDaysTimes(tripCache, currentVehicleState.getTrip().getId(),currentVehicleState.getTrip().getDirectionId(), + indices.getStopPathIndex(), nearestDay, time, + maxKalmanDaysToSearch.getValue(), maxKalmanDays.getValue()); + + if(lastDaysTimes!=null) + { + logger.debug("Kalman has " +lastDaysTimes.size()+ " historical values for : " +indices.toString()); + } + /* + * if we have enough data start using Kalman filter otherwise revert + * to extended class for prediction. + */ + if (lastDaysTimes != null && lastDaysTimes.size() >= minKalmanDays.getValue().intValue()) { + + logger.debug("Generating Kalman prediction for : "+indices.toString()); + + try { + + KalmanPrediction kalmanPrediction = new KalmanPrediction(); + + KalmanPredictionResult kalmanPredictionResult; + + Vehicle vehicle = new Vehicle(avlReport.getVehicleId()); + + VehicleStopDetail originDetail = new VehicleStopDetail(null, 0, vehicle); + TripSegment[] historical_segments_k = new TripSegment[lastDaysTimes.size()]; + for (int i = 0; i < lastDaysTimes.size() && i < maxKalmanDays.getValue(); i++) { + + logger.debug("Kalman is using historical value : "+lastDaysTimes.get(i) +" for : " + indices.toString()); + + VehicleStopDetail destinationDetail = new VehicleStopDetail(null, lastDaysTimes.get(i).getTravelTime(), + vehicle); + historical_segments_k[i] = new TripSegment(originDetail, destinationDetail); + } + + VehicleStopDetail destinationDetail_0_k_1 = new VehicleStopDetail(null, travelTimeDetails.getTravelTime(), vehicle); + + TripSegment ts_day_0_k_1 = new TripSegment(originDetail, destinationDetail_0_k_1); + + TripSegment last_vehicle_segment = ts_day_0_k_1; + + Indices previousVehicleIndices = new Indices(travelTimeDetails.getArrival()); + + Double last_prediction_error = lastVehiclePredictionError(kalmanErrorCache, previousVehicleIndices); + + logger.debug("Using error value: " + last_prediction_error +" found with vehicle id "+travelTimeDetails.getArrival().getVehicleId()+ " from: "+new KalmanErrorCacheKey(previousVehicleIndices).toString()); + + //TODO this should also display the detail of which vehicle it choose as the last one. + logger.debug("Using last vehicle value: " + travelTimeDetails + " for : "+ indices.toString()); + + kalmanPredictionResult = kalmanPrediction.predict(last_vehicle_segment, historical_segments_k, + last_prediction_error); + + long predictionTime = (long) kalmanPredictionResult.getResult(); + + logger.debug("Setting Kalman error value: " + kalmanPredictionResult.getFilterError() + " for : "+ new KalmanErrorCacheKey(indices).toString()); + + kalmanErrorCache.putErrorValue(indices, kalmanPredictionResult.getFilterError()); + + logger.debug("Using Kalman prediction: " + predictionTime + " instead of "+alternative+" prediction: " + + super.getTravelTimeForPath(indices, avlReport, vehicleState) +" for : " + indices.toString()); + + if(storeTravelTimeStopPathPredictions.getValue()) + { + PredictionForStopPath predictionForStopPath=new PredictionForStopPath(vehicleState.getVehicleId(), new Date(Core.getInstance().getSystemTime()), new Double(new Long(predictionTime).intValue()), indices.getTrip().getId(), indices.getStopPathIndex(), "KALMAN", true, null); + Core.getInstance().getDbLogger().add(predictionForStopPath); + StopPathPredictionCache.getInstance().putPrediction(predictionForStopPath); + } + return predictionTime; + + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + } + return super.getTravelTimeForPath(indices, avlReport, vehicleState); + } + + @Override + public long expectedTravelTimeFromMatchToEndOfStopPath(AvlReport avlReport, SpatialMatch match) { + + if(useKalmanForPartialStopPaths.getValue().booleanValue()) + { + VehicleStateManager vehicleStateManager = VehicleStateManager.getInstance(); + + VehicleState currentVehicleState = vehicleStateManager.getVehicleState(avlReport.getVehicleId()); + + long fulltime = this.getTravelTimeForPath(match.getIndices(), avlReport, currentVehicleState); + + double distanceAlongStopPath = match.getDistanceAlongStopPath(); + + double stopPathLength = + match.getStopPath().getLength(); + + long remainingtime = (long) (fulltime * ((stopPathLength-distanceAlongStopPath)/stopPathLength)); + + logger.debug("Using Kalman for first stop path {} with value {} instead of {}.", match.getIndices(), remainingtime, super.expectedTravelTimeFromMatchToEndOfStopPath(avlReport, match)); + + return remainingtime; + }else + { + return super.expectedTravelTimeFromMatchToEndOfStopPath(avlReport, match); + } + } + + private Double lastVehiclePredictionError(ErrorCache cache, Indices indices) { + + Double result = cache.getErrorValue(indices); + if(result!=null) + { + logger.debug("Kalman Error value : "+result +" for key: "+new KalmanErrorCacheKey(indices).toString()); + } + else + { + logger.debug("Kalman Error value set to default: "+initialErrorValue.getValue() +" for key: "+new KalmanErrorCacheKey(indices).toString()); + return initialErrorValue.getValue(); + } + return result; + } + + @Override + public long getStopTimeForPath(Indices indices, AvlReport avlReport, VehicleState vehicleState) { + long result=super.getStopTimeForPath(indices, avlReport, vehicleState); + + return result; + + } +} diff --git a/transitclock/src/main/java/org/transitclock/core/predictiongenerator/kalman/KalmanPredictionGeneratorImpl.java b/transitclock/src/main/java/org/transitclock/core/predictiongenerator/kalman/scheduled/KalmanPredictionGeneratorImpl.java similarity index 93% rename from transitclock/src/main/java/org/transitclock/core/predictiongenerator/kalman/KalmanPredictionGeneratorImpl.java rename to transitclock/src/main/java/org/transitclock/core/predictiongenerator/kalman/scheduled/KalmanPredictionGeneratorImpl.java index 25ae221b0..b7f28a071 100755 --- a/transitclock/src/main/java/org/transitclock/core/predictiongenerator/kalman/KalmanPredictionGeneratorImpl.java +++ b/transitclock/src/main/java/org/transitclock/core/predictiongenerator/kalman/scheduled/KalmanPredictionGeneratorImpl.java @@ -1,4 +1,4 @@ -package org.transitclock.core.predictiongenerator.kalman; +package org.transitclock.core.predictiongenerator.kalman.scheduled; import java.text.SimpleDateFormat; import java.util.Calendar; @@ -26,10 +26,15 @@ import org.transitclock.core.dataCache.TripDataHistoryCacheFactory; import org.transitclock.core.dataCache.TripDataHistoryCacheInterface; import org.transitclock.core.dataCache.VehicleStateManager; -import org.transitclock.core.dataCache.ehcache.TripDataHistoryCache; +import org.transitclock.core.dataCache.ehcache.scheduled.TripDataHistoryCache; import org.transitclock.core.dataCache.jcs.KalmanErrorCache; import org.transitclock.core.predictiongenerator.PredictionComponentElementsGenerator; import org.transitclock.core.predictiongenerator.average.scheduled.HistoricalAveragePredictionGeneratorImpl; +import org.transitclock.core.predictiongenerator.kalman.KalmanPrediction; +import org.transitclock.core.predictiongenerator.kalman.KalmanPredictionResult; +import org.transitclock.core.predictiongenerator.kalman.TripSegment; +import org.transitclock.core.predictiongenerator.kalman.Vehicle; +import org.transitclock.core.predictiongenerator.kalman.VehicleStopDetail; import org.transitclock.db.structs.AvlReport; import org.transitclock.db.structs.PredictionForStopPath; import org.transitclock.ipc.data.IpcPrediction; diff --git a/transitclock/src/main/java/org/transitclock/core/predictiongenerator/lastvehicle/LastVehiclePredictionGeneratorImpl.java b/transitclock/src/main/java/org/transitclock/core/predictiongenerator/lastvehicle/LastVehiclePredictionGeneratorImpl.java index 78f274265..5392ae467 100755 --- a/transitclock/src/main/java/org/transitclock/core/predictiongenerator/lastvehicle/LastVehiclePredictionGeneratorImpl.java +++ b/transitclock/src/main/java/org/transitclock/core/predictiongenerator/lastvehicle/LastVehiclePredictionGeneratorImpl.java @@ -18,7 +18,7 @@ import org.transitclock.core.dataCache.StopPathPredictionCache; import org.transitclock.core.dataCache.VehicleDataCache; import org.transitclock.core.dataCache.VehicleStateManager; -import org.transitclock.core.dataCache.ehcache.TripDataHistoryCache; +import org.transitclock.core.dataCache.ehcache.scheduled.TripDataHistoryCache; import org.transitclock.core.dataCache.scheduled.ScheduleBasedHistoricalAverageCache; import org.transitclock.core.predictiongenerator.PredictionComponentElementsGenerator; import org.transitclock.db.structs.AvlReport; diff --git a/transitclock/src/main/java/org/transitclock/core/predictiongenerator/rls/dwell/frequency/DwellTimePredictionGeneratorImpl.java b/transitclock/src/main/java/org/transitclock/core/predictiongenerator/rls/dwell/frequency/DwellTimePredictionGeneratorImpl.java new file mode 100644 index 000000000..4f8dda6a8 --- /dev/null +++ b/transitclock/src/main/java/org/transitclock/core/predictiongenerator/rls/dwell/frequency/DwellTimePredictionGeneratorImpl.java @@ -0,0 +1,81 @@ +package org.transitclock.core.predictiongenerator.rls.dwell.frequency; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.transitclock.core.Indices; +import org.transitclock.core.VehicleState; +import org.transitclock.core.dataCache.DwellTimeModelCacheFactory; +import org.transitclock.core.predictiongenerator.kalman.frequency.KalmanPredictionGeneratorImpl; +import org.transitclock.db.structs.AvlReport; +import org.transitclock.db.structs.Headway; + +/** + * @author Sean Og Crudden + * + * This is an experiment to see if headway can be used to better predict dwell time. Most of what + * I have read tells me it can but in conjunction with APC data and estimation of demand at stops. + * + * This is for frequency based services. + * + * + * + */ +public class DwellTimePredictionGeneratorImpl extends KalmanPredictionGeneratorImpl { + + private static final Logger logger = LoggerFactory.getLogger(DwellTimePredictionGeneratorImpl.class); + + @Override + public long getStopTimeForPath(Indices indices, AvlReport avlReport, VehicleState vehicleState) { + Long result=null; + try { + Headway headway = vehicleState.getHeadway(); + + if(headway!=null) + { + logger.debug("Headway at {} based on avl {} is {}.",indices, avlReport, headway); + + /* Change approach to use a RLS model. + */ + if(super.getStopTimeForPath(indices, avlReport, vehicleState)>0) + { + result = DwellTimeModelCacheFactory.getInstance().predictDwellTime(indices, headway); + + if(result==null) + { + logger.debug("Using scheduled value for dwell time as no RLS data available for {}.", indices); + result = super.getStopTimeForPath(indices, avlReport, vehicleState); + } + + + /* should never have a negative dwell time */ + if(result<0) + { + logger.debug("Predicted negative dwell time {} for {}.", result, indices); + result=0L; + } + + }else + { + logger.debug("Scheduled dwell time is less than 0 for {}.", indices); + result = super.getStopTimeForPath(indices, avlReport, vehicleState); + } + + logger.debug("Using dwell time {} for {} instead of {}. Headway for vehicle {} is {}",result,indices, super.getStopTimeForPath(indices, avlReport, vehicleState), vehicleState.getVehicleId(),headway ); + } + else + { + result = super.getStopTimeForPath(indices, avlReport, vehicleState); + logger.debug("Using dwell time {} for {} instead of {}. No headway.",result,indices, super.getStopTimeForPath(indices ,avlReport, vehicleState)); + } + + } catch (Exception e) { + + logger.error(e.getMessage(),e); + e.printStackTrace(); + + } + + return result; + } + +} diff --git a/transitclock/src/main/java/org/transitclock/core/predictiongenerator/rls/dwell/DwellTimePredictionGeneratorImpl.java b/transitclock/src/main/java/org/transitclock/core/predictiongenerator/rls/dwell/scheduled/DwellTimePredictionGeneratorImpl.java similarity index 94% rename from transitclock/src/main/java/org/transitclock/core/predictiongenerator/rls/dwell/DwellTimePredictionGeneratorImpl.java rename to transitclock/src/main/java/org/transitclock/core/predictiongenerator/rls/dwell/scheduled/DwellTimePredictionGeneratorImpl.java index c111278e6..c272610e1 100644 --- a/transitclock/src/main/java/org/transitclock/core/predictiongenerator/rls/dwell/DwellTimePredictionGeneratorImpl.java +++ b/transitclock/src/main/java/org/transitclock/core/predictiongenerator/rls/dwell/scheduled/DwellTimePredictionGeneratorImpl.java @@ -1,4 +1,4 @@ -package org.transitclock.core.predictiongenerator.rls.dwell; +package org.transitclock.core.predictiongenerator.rls.dwell.scheduled; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -9,7 +9,7 @@ import org.transitclock.core.VehicleState; import org.transitclock.core.dataCache.DwellTimeModelCacheFactory; import org.transitclock.core.dataCache.VehicleStateManager; -import org.transitclock.core.predictiongenerator.kalman.KalmanPredictionGeneratorImpl; +import org.transitclock.core.predictiongenerator.kalman.scheduled.KalmanPredictionGeneratorImpl; import org.transitclock.db.structs.AvlReport; import org.transitclock.db.structs.Headway; import org.transitclock.ipc.data.IpcPrediction; diff --git a/transitclock/src/main/java/org/transitclock/core/predictiongenerator/rls/dwell/TransitClockRLS.java b/transitclock/src/main/java/org/transitclock/core/predictiongenerator/rls/dwell/scheduled/TransitClockRLS.java similarity index 94% rename from transitclock/src/main/java/org/transitclock/core/predictiongenerator/rls/dwell/TransitClockRLS.java rename to transitclock/src/main/java/org/transitclock/core/predictiongenerator/rls/dwell/scheduled/TransitClockRLS.java index b7aafc017..05dd7c12c 100644 --- a/transitclock/src/main/java/org/transitclock/core/predictiongenerator/rls/dwell/TransitClockRLS.java +++ b/transitclock/src/main/java/org/transitclock/core/predictiongenerator/rls/dwell/scheduled/TransitClockRLS.java @@ -1,4 +1,4 @@ -package org.transitclock.core.predictiongenerator.rls.dwell; +package org.transitclock.core.predictiongenerator.rls.dwell.scheduled; import java.io.Serializable; diff --git a/transitclock/src/main/java/org/transitclock/custom/bullrunner/BullrunnerPlaybackModule.java b/transitclock/src/main/java/org/transitclock/custom/bullrunner/BullrunnerPlaybackModule.java new file mode 100644 index 000000000..15f48d462 --- /dev/null +++ b/transitclock/src/main/java/org/transitclock/custom/bullrunner/BullrunnerPlaybackModule.java @@ -0,0 +1,186 @@ +package org.transitclock.custom.bullrunner; + +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Collection; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.transitclock.avl.PollUrlAvlModule; +import org.transitclock.config.StringConfigValue; +import org.transitclock.db.structs.AvlReport; +import org.transitclock.db.structs.AvlReport.AssignmentType; +import org.transitclock.modules.Module; +import org.transitclock.utils.Time; + +/** + * @author scrudden + * Throw away module. Just used to read in archived Bullrunner data to work on Kalman predictions for frequency based services. + * + */ +public class BullrunnerPlaybackModule extends PollUrlAvlModule { + + private static StringConfigValue bullrunnerbasePlaybackUrl = + new StringConfigValue("transitclock.avl.bullrunnerbasePlaybackUrl", "http://transit.jadorno.com/bullrunner/gtfsrt/vehicle_positions", + "The URL of the historical json feed to use."); + + private static String getPlaybackStartTimeStr() { + return playbackStartTimeStr.getValue(); + } + + private static StringConfigValue playbackStartTimeStr = + new StringConfigValue("transitclock.avl.bullrunner.playbackStartTime", + "08-06-2018 12:00:00", + "Date and time of when to start the playback."); + + private static StringConfigValue playbackEndTimeStr = + new StringConfigValue("transitclock.avl.bullrunner.playbackEndTime", + "08-010-2018 23:00:00", + "Date and time of when to end the playback."); + + + + private static StringConfigValue runnerTestRoute = + new StringConfigValue("transitclock.avl.testroute", null, + "Route to test against."); + + private static final Logger logger = LoggerFactory + .getLogger(BullrunnerPlaybackModule.class); + + public BullrunnerPlaybackModule(String agencyId) { + super(agencyId); + + if(latesttime==null) + latesttime=parsePlaybackStartTime(playbackStartTimeStr.getValue()); + + } + Long latesttime=null; + + // If debugging feed and want to not actually process + // AVL reports to generate predictions and such then + // set shouldProcessAvl to false; + private static boolean shouldProcessAvl = true; + @Override + protected String getUrl() { + + return bullrunnerbasePlaybackUrl.getValue()+"?timestamp="+latesttime; + } + + public static void main(String[] args) { + // TODO Auto-generated method stub + Module.start("org.transitclock.custom.bullrunner.BullrunnerPlaybackModule"); + } + + @Override + protected Collection processData(InputStream in) throws Exception { + + String jsonStr = getJsonString(in); + Collection avlReportsReadIn = new ArrayList(); + try { + // Convert JSON string to a JSON object + JSONObject jsonObj = new JSONObject(jsonStr); + + JSONArray entities=(JSONArray) jsonObj.get("entity"); + JSONObject header=(JSONObject) jsonObj.get("header"); + + //JSONArray vehicles = entityObj.getJSONArray("vehicle"); + + for(int i=0;i(); + + } + + } + + + private static long parsePlaybackStartTime(String playbackStartTimeStr) { + try { + long playbackStartTime = Time.parse(playbackStartTimeStr).getTime(); + + // If specified time is in the future then reject. + if (playbackStartTime > System.currentTimeMillis()) { + logger.error("Playback start time \"{}\" specified by " + + "transitclock.avl.bullrunner.playbackStartTime parameter is in " + + "the future and therefore invalid!", + playbackStartTimeStr); + System.exit(-1); + } + + return playbackStartTime; + } catch (java.text.ParseException e) { + logger.error("Paramater -t \"{}\" specified by " + + "transitclock.avl.bullrunner.playbackStartTime parameter could not " + + "be parsed. Format must be \"MM-dd-yyyy HH:mm:ss\"", + playbackStartTimeStr); + System.exit(-1); + + // Will never be reached because the above state exits program but + // needed so compiler doesn't complain. + return -1; + } + } + private static long parsePlaybackEndTime(String playbackEndTimeStr) { + try { + long playbackEndTime = Time.parse(playbackEndTimeStr).getTime(); + + // If specified time is in the future then reject. + if (playbackEndTime > System.currentTimeMillis()) { + logger.error("Playback end time \"{}\" specified by " + + "transitclock.avl.playbackEndTime parameter is in " + + "the future and therefore invalid!", + playbackEndTimeStr); + System.exit(-1); + } + + return playbackEndTime; + } catch (java.text.ParseException e) { + logger.error("Paramater -t \"{}\" specified by " + + "transitclock.avl.playbackEndTime parameter could not " + + "be parsed. Format must be \"MM-dd-yyyy HH:mm:ss\"", + playbackEndTimeStr); + System.exit(-1); + + // Will never be reached because the above state exits program but + // needed so compiler doesn't complain. + return -1; + } + } +} From ff5fbee871b702f5cecd5166ff74ab92d1eaa00f Mon Sep 17 00:00:00 2001 From: scrudden Date: Sat, 25 Aug 2018 15:04:55 +0100 Subject: [PATCH 02/24] First running version of freq using kalman for travel times. --- .../java/org/transitclock/core/Indices.java | 13 ++++++- .../transitclock/core/TravelTimeDetails.java | 2 +- .../frequency/TripDataHistoryCache.java | 35 +++++++++++++++---- .../KalmanPredictionGeneratorImpl.java | 10 +++--- .../bullrunner/BullrunnerPlaybackModule.java | 13 ++++--- transitclock/src/main/resources/logback.xml | 30 +++------------- 6 files changed, 59 insertions(+), 44 deletions(-) diff --git a/transitclock/src/main/java/org/transitclock/core/Indices.java b/transitclock/src/main/java/org/transitclock/core/Indices.java index ee4fdc7d0..d6e2d3bc2 100644 --- a/transitclock/src/main/java/org/transitclock/core/Indices.java +++ b/transitclock/src/main/java/org/transitclock/core/Indices.java @@ -27,6 +27,7 @@ import org.transitclock.db.structs.Trip; import org.transitclock.db.structs.TripPattern; import org.transitclock.db.structs.VectorWithHeading; +import org.transitclock.gtfs.DbConfig; /** * This private class is for keeping track of the trip, path, and segment @@ -97,7 +98,17 @@ public Indices(SpatialMatch spatialMatch) { public Indices(ArrivalDeparture event) { - this.block=event.getBlock(); + + Block block=null; + if(event.getBlock()==null) + { + DbConfig dbConfig = Core.getInstance().getDbConfig(); + block=dbConfig.getBlock(event.getServiceId(), event.getBlockId()); + }else + { + block=event.getBlock(); + } + this.block=block; this.tripIndex = event.getTripIndex(); this.stopPathIndex = event.getStopPathIndex(); diff --git a/transitclock/src/main/java/org/transitclock/core/TravelTimeDetails.java b/transitclock/src/main/java/org/transitclock/core/TravelTimeDetails.java index c22760f93..feffcb6e6 100644 --- a/transitclock/src/main/java/org/transitclock/core/TravelTimeDetails.java +++ b/transitclock/src/main/java/org/transitclock/core/TravelTimeDetails.java @@ -13,7 +13,7 @@ public class TravelTimeDetails { private static final IntegerConfigValue maxTravelTime = new IntegerConfigValue( "transitclock.core.maxTravelTime", - 10 * Time.MS_PER_MIN, + 30 * Time.MS_PER_MIN, "This is a maximum allowed for travel between two stops. Used as a sanity check for cache and predictions."); private static final Logger logger = LoggerFactory diff --git a/transitclock/src/main/java/org/transitclock/core/dataCache/ehcache/frequency/TripDataHistoryCache.java b/transitclock/src/main/java/org/transitclock/core/dataCache/ehcache/frequency/TripDataHistoryCache.java index 298513b44..e07cd9789 100755 --- a/transitclock/src/main/java/org/transitclock/core/dataCache/ehcache/frequency/TripDataHistoryCache.java +++ b/transitclock/src/main/java/org/transitclock/core/dataCache/ehcache/frequency/TripDataHistoryCache.java @@ -30,6 +30,7 @@ import org.transitclock.core.dataCache.TripDataHistoryCacheFactory; import org.transitclock.core.dataCache.TripDataHistoryCacheInterface; import org.transitclock.core.dataCache.TripKey; +import org.transitclock.core.dataCache.frequency.FrequencyBasedHistoricalAverageCache; import org.transitclock.db.structs.ArrivalDeparture; import org.transitclock.db.structs.Block; import org.transitclock.db.structs.Trip; @@ -145,11 +146,13 @@ public void logCache(Logger logger) public List getTripHistory(TripKey tripKey) { //logger.debug(cache.toString()); - + logger.debug("Looking for TripDataHistoryCache cache element using key {}.", tripKey); + Element result = cache.get(tripKey); if(result!=null) { + logger.debug("Found TripDataHistoryCache cache element using key {}.", tripKey); return (List) result.getObjectValue(); } else @@ -165,7 +168,16 @@ public List getTripHistory(TripKey tripKey) { @SuppressWarnings("unchecked") synchronized public TripKey putArrivalDeparture(ArrivalDeparture arrivalDeparture) { - logger.debug("Putting :"+arrivalDeparture.toString() + " in TripDataHistoryCache cache."); + Block block=null; + if(arrivalDeparture.getBlock()==null) + { + DbConfig dbConfig = Core.getInstance().getDbConfig(); + block=dbConfig.getBlock(arrivalDeparture.getServiceId(), arrivalDeparture.getBlockId()); + }else + { + block=arrivalDeparture.getBlock(); + } + /* just put todays time in for last three days to aid development. This means it will kick in in 1 days rather than 3. Perhaps be a good way to start rather than using default transiTime method but I doubt it. */ int days_back=1; if(debug) @@ -182,15 +194,24 @@ synchronized public TripKey putArrivalDeparture(ArrivalDeparture arrivalDepartur Trip trip=dbConfig.getTrip(arrivalDeparture.getTripId()); - if(trip!=null) - { + // TODO need to set start time based on start of bucket + + Integer time=FrequencyBasedHistoricalAverageCache.secondsFromMidnight(arrivalDeparture.getDate(),2); + + time=FrequencyBasedHistoricalAverageCache.round(time, FrequencyBasedHistoricalAverageCache.getCacheIncrementsForFrequencyService()); + + if(trip!=null) + { + tripKey = new TripKey(arrivalDeparture.getTripId(), nearestDay, - trip.getStartTime()); + time); + + logger.debug("Putting :{} in TripDataHistoryCache cache using key {}.", arrivalDeparture, tripKey); + + List list = null; - List list = null; - Element result = cache.get(tripKey); if (result != null && result.getObjectValue() != null) { diff --git a/transitclock/src/main/java/org/transitclock/core/predictiongenerator/kalman/frequency/KalmanPredictionGeneratorImpl.java b/transitclock/src/main/java/org/transitclock/core/predictiongenerator/kalman/frequency/KalmanPredictionGeneratorImpl.java index 4744916fd..8d62395eb 100755 --- a/transitclock/src/main/java/org/transitclock/core/predictiongenerator/kalman/frequency/KalmanPredictionGeneratorImpl.java +++ b/transitclock/src/main/java/org/transitclock/core/predictiongenerator/kalman/frequency/KalmanPredictionGeneratorImpl.java @@ -47,15 +47,15 @@ public class KalmanPredictionGeneratorImpl extends HistoricalAveragePredictionGe * historical value. */ private static final IntegerConfigValue minKalmanDays = new IntegerConfigValue( - "transitclock.prediction.data.kalman.mindays", new Integer(3), + "transitclock.prediction.data.kalman.mindays", new Integer(2), "Min number of days trip data that needs to be available before Kalman prediciton is used instead of default transiTime prediction."); private static final IntegerConfigValue maxKalmanDays = new IntegerConfigValue( - "transitclock.prediction.data.kalman.maxdays", new Integer(3), + "transitclock.prediction.data.kalman.maxdays", new Integer(2), "Max number of historical days trips to include in Kalman prediction calculation."); private static final IntegerConfigValue maxKalmanDaysToSearch = new IntegerConfigValue( - "transitclock.prediction.data.kalman.maxdaystoseach", new Integer(21), + "transitclock.prediction.data.kalman.maxdaystoseach", new Integer(30), "Max number of days to look back for data. This will also be effected by how old the data in the cache is."); private static final DoubleConfigValue initialErrorValue = new DoubleConfigValue( @@ -109,12 +109,12 @@ public long getTravelTimeForPath(Indices indices, AvlReport avlReport, VehicleSt logger.debug("Kalman has last vehicle info for : " +indices.toString()+ " : "+travelTimeDetails); Date nearestDay = DateUtils.truncate(avlReport.getDate(), Calendar.DAY_OF_MONTH); - + List lastDaysTimes = lastDaysTimes(tripCache, currentVehicleState.getTrip().getId(),currentVehicleState.getTrip().getDirectionId(), indices.getStopPathIndex(), nearestDay, time, maxKalmanDaysToSearch.getValue(), maxKalmanDays.getValue()); - if(lastDaysTimes!=null) + if(lastDaysTimes!=null&&lastDaysTimes.size()>0) { logger.debug("Kalman has " +lastDaysTimes.size()+ " historical values for : " +indices.toString()); } diff --git a/transitclock/src/main/java/org/transitclock/custom/bullrunner/BullrunnerPlaybackModule.java b/transitclock/src/main/java/org/transitclock/custom/bullrunner/BullrunnerPlaybackModule.java index 15f48d462..49f4d3dff 100644 --- a/transitclock/src/main/java/org/transitclock/custom/bullrunner/BullrunnerPlaybackModule.java +++ b/transitclock/src/main/java/org/transitclock/custom/bullrunner/BullrunnerPlaybackModule.java @@ -9,6 +9,7 @@ import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.transitclock.applications.Core; import org.transitclock.avl.PollUrlAvlModule; import org.transitclock.config.StringConfigValue; import org.transitclock.db.structs.AvlReport; @@ -38,7 +39,7 @@ private static String getPlaybackStartTimeStr() { private static StringConfigValue playbackEndTimeStr = new StringConfigValue("transitclock.avl.bullrunner.playbackEndTime", - "08-010-2018 23:00:00", + "08-09-2018 23:00:00", "Date and time of when to end the playback."); @@ -100,17 +101,21 @@ protected Collection processData(InputStream in) throws Exception { { //String blockid=vehicle.getString("block_id"); long timestamp=header.getLong("timestamp"); - + + Core.getInstance().setSystemTime(latesttime); + + // Create the AvlReport AvlReport avlReport = - new AvlReport(vehicleIdentity.getString("id"), timestamp*1000, vehiclePosition.getDouble("latitude"), vehiclePosition.getDouble("longitude"), Float.NaN, - (float) vehiclePosition.getInt("bearing"), "BusLoc"); + new AvlReport(vehicleIdentity.getString("id"), latesttime, vehiclePosition.getDouble("latitude"), vehiclePosition.getDouble("longitude"), Float.NaN, + (float) vehiclePosition.getInt("bearing"), "BullrunnerArchive"); // Actually set the assignment avlReport.setAssignment(vehicleTrip.getString("routeId"), AssignmentType.ROUTE_ID); logger.debug("From BullrunnerPlaybackModule {}", avlReport); + System.out.println(avlReport); if (shouldProcessAvl) { avlReportsReadIn.add(avlReport); diff --git a/transitclock/src/main/resources/logback.xml b/transitclock/src/main/resources/logback.xml index e4db83feb..4d7686dce 100755 --- a/transitclock/src/main/resources/logback.xml +++ b/transitclock/src/main/resources/logback.xml @@ -387,33 +387,11 @@ - - - - - - - - - - - - - - - + @@ -501,9 +479,9 @@ level="warn" additivity="false"> - - + Date: Sun, 2 Sep 2018 19:00:10 +0100 Subject: [PATCH 03/24] Added code to predict dwell times for frequency based services using RLS. --- .../transitclock/configData/AvlConfig.java | 2 +- .../core/dataCache/DwellTimeCacheKey.java | 74 ------- .../dataCache/DwellTimeModelCacheFactory.java | 2 +- .../DwellTimeModelCacheInterface.java | 5 +- .../core/dataCache/StopPathCacheKey.java | 1 + .../jcs/frequency/DwellTimeModelCache.java | 206 ++++++++++++++++++ .../{ => scheduled}/DwellTimeModelCache.java | 25 +-- .../DwellTimePredictionGeneratorImpl.java | 15 +- .../DwellTimePredictionGeneratorImpl.java | 7 +- 9 files changed, 239 insertions(+), 98 deletions(-) delete mode 100644 transitclock/src/main/java/org/transitclock/core/dataCache/DwellTimeCacheKey.java create mode 100644 transitclock/src/main/java/org/transitclock/core/dataCache/jcs/frequency/DwellTimeModelCache.java rename transitclock/src/main/java/org/transitclock/core/dataCache/jcs/{ => scheduled}/DwellTimeModelCache.java (87%) diff --git a/transitclock/src/main/java/org/transitclock/configData/AvlConfig.java b/transitclock/src/main/java/org/transitclock/configData/AvlConfig.java index a68783199..ce2827e37 100644 --- a/transitclock/src/main/java/org/transitclock/configData/AvlConfig.java +++ b/transitclock/src/main/java/org/transitclock/configData/AvlConfig.java @@ -52,7 +52,7 @@ public static int getSecondsBetweenAvlFeedPolling() { return secondsBetweenAvlFeedPolling.getValue(); } private static IntegerConfigValue secondsBetweenAvlFeedPolling = - new IntegerConfigValue("transitclock.avl.feedPollingRateSecs", 5, + new IntegerConfigValue("transitclock.avl.feedPollingRateSecs", -1, "How frequently an AVL feed should be polled for new data."); /** diff --git a/transitclock/src/main/java/org/transitclock/core/dataCache/DwellTimeCacheKey.java b/transitclock/src/main/java/org/transitclock/core/dataCache/DwellTimeCacheKey.java deleted file mode 100644 index 5ad238b29..000000000 --- a/transitclock/src/main/java/org/transitclock/core/dataCache/DwellTimeCacheKey.java +++ /dev/null @@ -1,74 +0,0 @@ -package org.transitclock.core.dataCache; - -import java.io.Serializable; - -import org.transitclock.core.Indices; - -public class DwellTimeCacheKey implements Serializable { - /** - * - */ - private static final long serialVersionUID = 4967988031829738238L; - private String tripId; - private Integer stopPathIndex; - public DwellTimeCacheKey(String tripId, Integer stopPathIndex) { - this.tripId = tripId; - this.stopPathIndex = stopPathIndex; - } - - public DwellTimeCacheKey(Indices indices) { - - this.tripId = indices.getTrip().getId(); - this.stopPathIndex = indices.getStopPathIndex(); - } - public String getTripId() { - return tripId; - } - public void setTripId(String tripId) { - this.tripId = tripId; - } - public Integer getStopPathIndex() { - return stopPathIndex; - } - public void setStopPathIndex(Integer stopPathIndex) { - this.stopPathIndex = stopPathIndex; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((stopPathIndex == null) ? 0 : stopPathIndex.hashCode()); - result = prime * result + ((tripId == null) ? 0 : tripId.hashCode()); - return result; - } - - @Override - public String toString() { - return "DwellTimeCacheKey [tripId=" + tripId + ", stopPathIndex=" + stopPathIndex + "]"; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - DwellTimeCacheKey other = (DwellTimeCacheKey) obj; - if (stopPathIndex == null) { - if (other.stopPathIndex != null) - return false; - } else if (!stopPathIndex.equals(other.stopPathIndex)) - return false; - if (tripId == null) { - if (other.tripId != null) - return false; - } else if (!tripId.equals(other.tripId)) - return false; - return true; - } - - -} diff --git a/transitclock/src/main/java/org/transitclock/core/dataCache/DwellTimeModelCacheFactory.java b/transitclock/src/main/java/org/transitclock/core/dataCache/DwellTimeModelCacheFactory.java index 19293d8ed..66940933e 100644 --- a/transitclock/src/main/java/org/transitclock/core/dataCache/DwellTimeModelCacheFactory.java +++ b/transitclock/src/main/java/org/transitclock/core/dataCache/DwellTimeModelCacheFactory.java @@ -10,7 +10,7 @@ public class DwellTimeModelCacheFactory { private static StringConfigValue className = new StringConfigValue("transitclock.core.cache.dwellTimeModelCache", - "org.transitclock.core.dataCache.jcs.DwellTimeModelCache", + "org.transitclock.core.dataCache.jcs.scheduled.DwellTimeModelCache", "Specifies the class used to cache RLS data for a stop."); private static DwellTimeModelCacheInterface singleton = null; diff --git a/transitclock/src/main/java/org/transitclock/core/dataCache/DwellTimeModelCacheInterface.java b/transitclock/src/main/java/org/transitclock/core/dataCache/DwellTimeModelCacheInterface.java index 9ce6a04d2..d169ebeeb 100644 --- a/transitclock/src/main/java/org/transitclock/core/dataCache/DwellTimeModelCacheInterface.java +++ b/transitclock/src/main/java/org/transitclock/core/dataCache/DwellTimeModelCacheInterface.java @@ -1,14 +1,13 @@ package org.transitclock.core.dataCache; -import org.transitclock.core.Indices; import org.transitclock.db.structs.ArrivalDeparture; import org.transitclock.db.structs.Headway; public interface DwellTimeModelCacheInterface { - void addSample(Indices indices, Headway headway, long dwellTime); + void addSample(ArrivalDeparture event, Headway headway, long dwellTime); void addSample(ArrivalDeparture departure); - Long predictDwellTime(Indices indices, Headway headway); + Long predictDwellTime(StopPathCacheKey cacheKey, Headway headway); } diff --git a/transitclock/src/main/java/org/transitclock/core/dataCache/StopPathCacheKey.java b/transitclock/src/main/java/org/transitclock/core/dataCache/StopPathCacheKey.java index aac34cb57..2e42b06b0 100755 --- a/transitclock/src/main/java/org/transitclock/core/dataCache/StopPathCacheKey.java +++ b/transitclock/src/main/java/org/transitclock/core/dataCache/StopPathCacheKey.java @@ -33,6 +33,7 @@ public StopPathCacheKey(String tripId, Integer stopPathIndex) this.tripId = tripId; this.stopPathIndex = stopPathIndex; this.travelTime=true; + this.startTime=null; } public StopPathCacheKey(String tripId, Integer stopPathIndex, boolean travelTime) { diff --git a/transitclock/src/main/java/org/transitclock/core/dataCache/jcs/frequency/DwellTimeModelCache.java b/transitclock/src/main/java/org/transitclock/core/dataCache/jcs/frequency/DwellTimeModelCache.java new file mode 100644 index 000000000..de10fc1e4 --- /dev/null +++ b/transitclock/src/main/java/org/transitclock/core/dataCache/jcs/frequency/DwellTimeModelCache.java @@ -0,0 +1,206 @@ +package org.transitclock.core.dataCache.jcs.frequency; + +import java.util.List; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.jcs.JCS; +import org.apache.commons.jcs.access.CacheAccess; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.transitclock.applications.Core; +import org.transitclock.config.DoubleConfigValue; +import org.transitclock.config.IntegerConfigValue; +import org.transitclock.config.LongConfigValue; +import org.transitclock.core.Indices; +import org.transitclock.core.dataCache.StopArrivalDepartureCacheFactory; +import org.transitclock.core.dataCache.StopArrivalDepartureCacheKey; +import org.transitclock.core.dataCache.StopPathCacheKey; +import org.transitclock.core.dataCache.frequency.FrequencyBasedHistoricalAverageCache; +import org.transitclock.core.predictiongenerator.rls.dwell.scheduled.TransitClockRLS; +import org.transitclock.db.structs.ArrivalDeparture; +import org.transitclock.db.structs.Block; +import org.transitclock.db.structs.Headway; +import org.transitclock.gtfs.DbConfig; +import org.transitclock.utils.Time; + +public class DwellTimeModelCache implements org.transitclock.core.dataCache.DwellTimeModelCacheInterface { + + final private static String cacheName = "DwellTimeModelCache"; + + private static IntegerConfigValue maxDwellTimeAllowedInModel = new IntegerConfigValue("org.transitclock.core.dataCache.jcs.maxDwellTimeAllowedInModel", 2 * Time.MS_PER_MIN, "Max dwell time to be considered in dwell RLS algotithm."); + private static LongConfigValue maxHeadwayAllowedInModel = new LongConfigValue("org.transitclock.core.dataCache.jcs.maxHeadwayAllowedInModel", 1*Time.MS_PER_HOUR, "Max headway to be considered in dwell RLS algotithm."); + + private static DoubleConfigValue lambda = new DoubleConfigValue("org.transitclock.core.dataCache.jcs.lambda", 0.75, "This sets the rate at which the RLS algorithm forgets old values. Value are between 0 and 1. With 0 being the most forgetful."); + + private CacheAccess cache = null; + + private static final Logger logger = LoggerFactory.getLogger(DwellTimeModelCache.class); + + public DwellTimeModelCache() { + cache = JCS.getInstance(cacheName); + } + @Override + synchronized public void addSample(ArrivalDeparture event, Headway headway, long dwellTime) { + + Integer time=FrequencyBasedHistoricalAverageCache.secondsFromMidnight(event.getFreqStartTime(),2); + + time=FrequencyBasedHistoricalAverageCache.round(time, FrequencyBasedHistoricalAverageCache.getCacheIncrementsForFrequencyService()); + + StopPathCacheKey key=new StopPathCacheKey(event.getTripId(), event.getStopPathIndex(), false, new Long(time)); + + TransitClockRLS rls = null; + if(cache.get(key)!=null) + { + rls=cache.get(key); + + double[] x = new double[1]; + x[0]=headway.getHeadway(); + + double y = Math.log10(dwellTime); + + + double[] arg0 = new double[1]; + arg0[0]=headway.getHeadway(); + if(rls.getRls()!=null) + { + double prediction = Math.pow(10,rls.getRls().predict(arg0)); + + logger.debug("Predicted dwell: "+prediction + " for: "+key + " based on headway: "+TimeUnit.MILLISECONDS.toMinutes((long) headway.getHeadway())+" mins"); + + logger.debug("Actual dwell: "+ dwellTime + " for: "+key + " based on headway: "+TimeUnit.MILLISECONDS.toMinutes((long) headway.getHeadway())+" mins"); + } + + rls.addSample(headway.getHeadway(), Math.log10(dwellTime)); + if(rls.getRls()!=null) + { + double prediction = Math.pow(10,rls.getRls().predict(arg0)); + + logger.debug("Predicted dwell after: "+ prediction + " for: "+key+ " with samples: "+rls.numSamples()); + } + }else + { + + rls=new TransitClockRLS(lambda.getValue()); + rls.addSample(headway.getHeadway(), Math.log10(dwellTime)); + } + cache.put(key,rls); + } + + @Override + public void addSample(ArrivalDeparture departure) { + if(departure!=null && !departure.isArrival()) + { + Block block=null; + if(departure.getBlock()==null) + { + DbConfig dbConfig = Core.getInstance().getDbConfig(); + block=dbConfig.getBlock(departure.getServiceId(), departure.getBlockId()); + }else + { + block=departure.getBlock(); + } + + Indices indices = new Indices(departure); + StopArrivalDepartureCacheKey key= new StopArrivalDepartureCacheKey(departure.getStopId(), departure.getDate()); + List stopData = StopArrivalDepartureCacheFactory.getInstance().getStopHistory(key); + + if(stopData!=null && stopData.size()>1) + { + ArrivalDeparture arrival=findArrival(stopData, departure); + if(arrival!=null) + { + ArrivalDeparture previousArrival=findPreviousArrival(stopData, arrival); + if(arrival!=null&&previousArrival!=null) + { + Headway headway=new Headway(); + headway.setHeadway(arrival.getTime()-previousArrival.getTime()); + long dwelltime=departure.getTime()-arrival.getTime(); + headway.setTripId(arrival.getTripId()); + + // Negative dwell times are errors in data so do not include. + // TODO not sure if it should ignore zero values. + if(dwelltime>=0) + { + /* Leave out silly values as they are most likely errors or unusual circumstance. */ + if(dwelltime stopData, ArrivalDeparture arrival) { + for(ArrivalDeparture event:stopData) + { + if(event.isArrival()) + { + if(!event.getVehicleId().equals(arrival.getVehicleId())) + { + if(!event.getTripId().equals(arrival.getTripId())) + { + if(event.getStopId().equals(arrival.getStopId())) + { + if(event.getTime() stopData, ArrivalDeparture departure) { + + for(ArrivalDeparture event:stopData) + { + if(event.isArrival()) + { + if(event.getStopId().equals(departure.getStopId())) + { + if(event.getVehicleId().equals(departure.getVehicleId())) + { + if(event.getTripId().equals(departure.getTripId())) + { + return event; + } + } + } + } + } + return null; + } + @Override + public Long predictDwellTime(StopPathCacheKey cacheKey, Headway headway) { + //TODO null should be the start time of the freq bucket. + Integer time=FrequencyBasedHistoricalAverageCache.secondsFromMidnight(null ,2); + + time=FrequencyBasedHistoricalAverageCache.round(time, FrequencyBasedHistoricalAverageCache.getCacheIncrementsForFrequencyService()); + + TransitClockRLS rls=cache.get(cacheKey); + if(rls!=null&&rls.getRls()!=null) + { + double[] arg0 = new double[1]; + arg0[0]=headway.getHeadway(); + rls.getRls().predict(arg0); + return (long) Math.pow(10, rls.getRls().predict(arg0)); + }else + { + return null; + } + } + public static void main(String[] args) + { + double startvalue=1000; + double result1 = Math.log10(startvalue); + double result2 = Math.pow(10, result1); + if(startvalue==result2) + System.out.println("As expected they are the same."); + } + +} diff --git a/transitclock/src/main/java/org/transitclock/core/dataCache/jcs/DwellTimeModelCache.java b/transitclock/src/main/java/org/transitclock/core/dataCache/jcs/scheduled/DwellTimeModelCache.java similarity index 87% rename from transitclock/src/main/java/org/transitclock/core/dataCache/jcs/DwellTimeModelCache.java rename to transitclock/src/main/java/org/transitclock/core/dataCache/jcs/scheduled/DwellTimeModelCache.java index 84a9532fc..aa8a9af3e 100644 --- a/transitclock/src/main/java/org/transitclock/core/dataCache/jcs/DwellTimeModelCache.java +++ b/transitclock/src/main/java/org/transitclock/core/dataCache/jcs/scheduled/DwellTimeModelCache.java @@ -1,4 +1,4 @@ -package org.transitclock.core.dataCache.jcs; +package org.transitclock.core.dataCache.jcs.scheduled; import java.util.List; import java.util.concurrent.TimeUnit; @@ -11,14 +11,10 @@ import org.transitclock.config.DoubleConfigValue; import org.transitclock.config.IntegerConfigValue; import org.transitclock.config.LongConfigValue; -import org.transitclock.config.StringConfigValue; -import org.transitclock.core.HeadwayDetails; import org.transitclock.core.Indices; -import org.transitclock.core.TravelTimes; -import org.transitclock.core.dataCache.DwellTimeCacheKey; import org.transitclock.core.dataCache.StopArrivalDepartureCacheFactory; import org.transitclock.core.dataCache.StopArrivalDepartureCacheKey; -import org.transitclock.core.predictiongenerator.rls.dwell.scheduled.DwellTimePredictionGeneratorImpl; +import org.transitclock.core.dataCache.StopPathCacheKey; import org.transitclock.core.predictiongenerator.rls.dwell.scheduled.TransitClockRLS; import org.transitclock.db.structs.ArrivalDeparture; import org.transitclock.db.structs.Block; @@ -26,8 +22,6 @@ import org.transitclock.gtfs.DbConfig; import org.transitclock.utils.Time; -import smile.regression.RLS; - public class DwellTimeModelCache implements org.transitclock.core.dataCache.DwellTimeModelCacheInterface { final private static String cacheName = "DwellTimeModelCache"; @@ -37,7 +31,7 @@ public class DwellTimeModelCache implements org.transitclock.core.dataCache.Dwel private static DoubleConfigValue lambda = new DoubleConfigValue("org.transitclock.core.dataCache.jcs.lambda", 0.75, "This sets the rate at which the RLS algorithm forgets old values. Value are between 0 and 1. With 0 being the most forgetful."); - private CacheAccess cache = null; + private CacheAccess cache = null; private static final Logger logger = LoggerFactory.getLogger(DwellTimeModelCache.class); @@ -45,9 +39,9 @@ public DwellTimeModelCache() { cache = JCS.getInstance(cacheName); } @Override - synchronized public void addSample(Indices indices, Headway headway, long dwellTime) { + synchronized public void addSample(ArrivalDeparture event, Headway headway, long dwellTime) { - DwellTimeCacheKey key=new DwellTimeCacheKey(headway.getTripId(), indices.getStopPathIndex()); + StopPathCacheKey key=new StopPathCacheKey(headway.getTripId(), event.getStopPathIndex()); TransitClockRLS rls = null; @@ -127,7 +121,7 @@ public void addSample(ArrivalDeparture departure) { if(dwelltime stopData, ArrivalDep return null; } @Override - public Long predictDwellTime(Indices indices, Headway headway) { - - DwellTimeCacheKey key=new DwellTimeCacheKey(indices); - TransitClockRLS rls=cache.get(key); + public Long predictDwellTime(StopPathCacheKey cacheKey, Headway headway) { + + TransitClockRLS rls=cache.get(cacheKey); if(rls!=null&&rls.getRls()!=null) { double[] arg0 = new double[1]; diff --git a/transitclock/src/main/java/org/transitclock/core/predictiongenerator/rls/dwell/frequency/DwellTimePredictionGeneratorImpl.java b/transitclock/src/main/java/org/transitclock/core/predictiongenerator/rls/dwell/frequency/DwellTimePredictionGeneratorImpl.java index 4f8dda6a8..81457d5b5 100644 --- a/transitclock/src/main/java/org/transitclock/core/predictiongenerator/rls/dwell/frequency/DwellTimePredictionGeneratorImpl.java +++ b/transitclock/src/main/java/org/transitclock/core/predictiongenerator/rls/dwell/frequency/DwellTimePredictionGeneratorImpl.java @@ -1,10 +1,14 @@ package org.transitclock.core.predictiongenerator.rls.dwell.frequency; +import java.util.Date; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.transitclock.core.Indices; import org.transitclock.core.VehicleState; import org.transitclock.core.dataCache.DwellTimeModelCacheFactory; +import org.transitclock.core.dataCache.StopPathCacheKey; +import org.transitclock.core.dataCache.frequency.FrequencyBasedHistoricalAverageCache; import org.transitclock.core.predictiongenerator.kalman.frequency.KalmanPredictionGeneratorImpl; import org.transitclock.db.structs.AvlReport; import org.transitclock.db.structs.Headway; @@ -37,8 +41,15 @@ public long getStopTimeForPath(Indices indices, AvlReport avlReport, VehicleSta /* Change approach to use a RLS model. */ if(super.getStopTimeForPath(indices, avlReport, vehicleState)>0) - { - result = DwellTimeModelCacheFactory.getInstance().predictDwellTime(indices, headway); + { + // TODO Would be more correct to use the start time of the trip. + Integer time=FrequencyBasedHistoricalAverageCache.secondsFromMidnight(new Date(avlReport.getTime()),2); + + time=FrequencyBasedHistoricalAverageCache.round(time, FrequencyBasedHistoricalAverageCache.getCacheIncrementsForFrequencyService()); + + StopPathCacheKey cacheKey=new StopPathCacheKey(indices.getTrip().getId(), indices.getStopPathIndex(), false, new Long(time)); + + result = DwellTimeModelCacheFactory.getInstance().predictDwellTime(cacheKey, headway); if(result==null) { diff --git a/transitclock/src/main/java/org/transitclock/core/predictiongenerator/rls/dwell/scheduled/DwellTimePredictionGeneratorImpl.java b/transitclock/src/main/java/org/transitclock/core/predictiongenerator/rls/dwell/scheduled/DwellTimePredictionGeneratorImpl.java index c272610e1..e66f955c2 100644 --- a/transitclock/src/main/java/org/transitclock/core/predictiongenerator/rls/dwell/scheduled/DwellTimePredictionGeneratorImpl.java +++ b/transitclock/src/main/java/org/transitclock/core/predictiongenerator/rls/dwell/scheduled/DwellTimePredictionGeneratorImpl.java @@ -8,6 +8,7 @@ import org.transitclock.core.TemporalDifference; import org.transitclock.core.VehicleState; import org.transitclock.core.dataCache.DwellTimeModelCacheFactory; +import org.transitclock.core.dataCache.StopPathCacheKey; import org.transitclock.core.dataCache.VehicleStateManager; import org.transitclock.core.predictiongenerator.kalman.scheduled.KalmanPredictionGeneratorImpl; import org.transitclock.db.structs.AvlReport; @@ -41,7 +42,11 @@ public long getStopTimeForPath(Indices indices, AvlReport avlReport, VehicleSta */ if(super.getStopTimeForPath(indices, avlReport, vehicleState)>0) { - result = DwellTimeModelCacheFactory.getInstance().predictDwellTime(indices, headway); + + StopPathCacheKey cacheKey=new StopPathCacheKey(indices.getTrip().getId(), indices.getStopPathIndex(), false); + + + result = DwellTimeModelCacheFactory.getInstance().predictDwellTime(cacheKey, headway); if(result==null) { From 411574aec5ea733c4acf871a254a96174e2b65e7 Mon Sep 17 00:00:00 2001 From: scrudden Date: Sun, 2 Sep 2018 19:39:24 +0100 Subject: [PATCH 04/24] reset polling rate. --- .../src/main/java/org/transitclock/configData/AvlConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transitclock/src/main/java/org/transitclock/configData/AvlConfig.java b/transitclock/src/main/java/org/transitclock/configData/AvlConfig.java index ce2827e37..a68783199 100644 --- a/transitclock/src/main/java/org/transitclock/configData/AvlConfig.java +++ b/transitclock/src/main/java/org/transitclock/configData/AvlConfig.java @@ -52,7 +52,7 @@ public static int getSecondsBetweenAvlFeedPolling() { return secondsBetweenAvlFeedPolling.getValue(); } private static IntegerConfigValue secondsBetweenAvlFeedPolling = - new IntegerConfigValue("transitclock.avl.feedPollingRateSecs", -1, + new IntegerConfigValue("transitclock.avl.feedPollingRateSecs", 5, "How frequently an AVL feed should be polled for new data."); /** From 0cc8b119d488487599bf8eff432aeb76e4a8e091 Mon Sep 17 00:00:00 2001 From: scrudden Date: Sat, 8 Sep 2018 10:46:46 +0100 Subject: [PATCH 05/24] Broader check to see if vehicle has passed first stop. --- .../java/org/transitclock/core/VehicleState.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/transitclock/src/main/java/org/transitclock/core/VehicleState.java b/transitclock/src/main/java/org/transitclock/core/VehicleState.java index e807d16ed..eb6640290 100644 --- a/transitclock/src/main/java/org/transitclock/core/VehicleState.java +++ b/transitclock/src/main/java/org/transitclock/core/VehicleState.java @@ -97,7 +97,6 @@ public class VehicleState { private Headway headway=null; - public Headway getHeadway() { return headway; } @@ -380,8 +379,8 @@ public Long getTripStartTime(Integer tripCounter) */ public void incrementTripCounter(ArrivalDeparture event) { - VehicleState vehicleState = VehicleStateManager.getInstance().getVehicleState(event.getVehicleId()); - + VehicleState vehicleState = VehicleStateManager.getInstance().getVehicleState(event.getVehicleId()); + if(event.getStopPathIndex()==0) { if(event.isArrival()) @@ -391,8 +390,13 @@ public void incrementTripCounter(ArrivalDeparture event) }else { logger.debug("Not arrival so not incrementing trip counter : {}", event); - } - } + } + }else + if(this.getPreviousMatch().getStopPathIndex()>event.getStopPathIndex()) + { + vehicleState.incrementTripCounter(); + logger.debug("Setting vehicle counter to : {} because of event : {}",vehicleState.getTripCounter(), event); + } } /** * Returns the current Trip for the vehicle. Returns null if there is not From 46d710310e298d2195bc30c8bf9f77c035022f8a Mon Sep 17 00:00:00 2001 From: scrudden Date: Sun, 9 Sep 2018 12:25:00 +0100 Subject: [PATCH 06/24] Fix compile is with merge. --- .../core/dataCache/jcs/scheduled/DwellTimeModelCache.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/transitclock/src/main/java/org/transitclock/core/dataCache/jcs/scheduled/DwellTimeModelCache.java b/transitclock/src/main/java/org/transitclock/core/dataCache/jcs/scheduled/DwellTimeModelCache.java index 252de8023..eb38728ee 100644 --- a/transitclock/src/main/java/org/transitclock/core/dataCache/jcs/scheduled/DwellTimeModelCache.java +++ b/transitclock/src/main/java/org/transitclock/core/dataCache/jcs/scheduled/DwellTimeModelCache.java @@ -119,7 +119,6 @@ public void addSample(ArrivalDeparture departure) { long dwelltime=departure.getTime()-arrival.getTime(); headway.setTripId(arrival.getTripId()); - /* Leave out silly values as they are most likely errors or unusual circumstance. */ /* TODO Should abstract this behind an anomaly detention interface/Factory */ if(dwelltime minHeadwayAllowedInModel.getValue()) { - addSample(indices, headway,dwelltime); + addSample(departure,headway,dwelltime); } } From 92d864f8634e246ef7d20aa04e591a0e1256d0d9 Mon Sep 17 00:00:00 2001 From: scrudden Date: Sun, 9 Sep 2018 15:04:03 +0100 Subject: [PATCH 07/24] Fix issue #107 --- .../java/org/transitclock/api/rootResources/TransitimeApi.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transitclockApi/src/main/java/org/transitclock/api/rootResources/TransitimeApi.java b/transitclockApi/src/main/java/org/transitclock/api/rootResources/TransitimeApi.java index 3ecdef5a2..54f96d072 100644 --- a/transitclockApi/src/main/java/org/transitclock/api/rootResources/TransitimeApi.java +++ b/transitclockApi/src/main/java/org/transitclock/api/rootResources/TransitimeApi.java @@ -774,7 +774,7 @@ public Response getRoutes(@BeanParam StandardParameters stdParameters, + "and paths such that it can be drawn in a map.",tags= {"base data","route"}) public Response getRouteDetails(@BeanParam StandardParameters stdParameters, @Parameter(description="List of routeId or routeShortName. Example: r=1&r=2" ,required=false) - List routeIdsOrShortNames, + @QueryParam(value = "r") List routeIdsOrShortNames, @Parameter(description="If set then only the shape for specified direction is marked as being for the UI." ,required=false) @QueryParam(value = "d") String directionId, @Parameter(description="If set then only this stop and the remaining ones on " From 5d1fd8a6cc47abdb50d883e94f15918ea33f3889 Mon Sep 17 00:00:00 2001 From: scrudden Date: Wed, 12 Sep 2018 18:40:42 +0100 Subject: [PATCH 08/24] Differentiate trip by freqStartTime. --- .../frequency/TripDataHistoryCache.java | 64 ++++++++++--------- .../jcs/frequency/DwellTimeModelCache.java | 24 +++++-- 2 files changed, 52 insertions(+), 36 deletions(-) diff --git a/transitclock/src/main/java/org/transitclock/core/dataCache/ehcache/frequency/TripDataHistoryCache.java b/transitclock/src/main/java/org/transitclock/core/dataCache/ehcache/frequency/TripDataHistoryCache.java index e07cd9789..ddc2cbaeb 100755 --- a/transitclock/src/main/java/org/transitclock/core/dataCache/ehcache/frequency/TripDataHistoryCache.java +++ b/transitclock/src/main/java/org/transitclock/core/dataCache/ehcache/frequency/TripDataHistoryCache.java @@ -195,40 +195,44 @@ synchronized public TripKey putArrivalDeparture(ArrivalDeparture arrivalDepartur Trip trip=dbConfig.getTrip(arrivalDeparture.getTripId()); // TODO need to set start time based on start of bucket - - Integer time=FrequencyBasedHistoricalAverageCache.secondsFromMidnight(arrivalDeparture.getDate(),2); - - time=FrequencyBasedHistoricalAverageCache.round(time, FrequencyBasedHistoricalAverageCache.getCacheIncrementsForFrequencyService()); - - if(trip!=null) - { - - - tripKey = new TripKey(arrivalDeparture.getTripId(), - nearestDay, - time); - - logger.debug("Putting :{} in TripDataHistoryCache cache using key {}.", arrivalDeparture, tripKey); + if(arrivalDeparture.getFreqStartTime()!=null) + { + Integer time=FrequencyBasedHistoricalAverageCache.secondsFromMidnight(arrivalDeparture.getFreqStartTime(),2); - List list = null; + time=FrequencyBasedHistoricalAverageCache.round(time, FrequencyBasedHistoricalAverageCache.getCacheIncrementsForFrequencyService()); - Element result = cache.get(tripKey); - - if (result != null && result.getObjectValue() != null) { - list = (List) result.getObjectValue(); - cache.remove(tripKey); - } else { - list = new ArrayList(); + if(trip!=null) + { + + + tripKey = new TripKey(arrivalDeparture.getTripId(), + nearestDay, + time); + + logger.debug("Putting :{} in TripDataHistoryCache cache using key {}.", arrivalDeparture, tripKey); + + List list = null; + + Element result = cache.get(tripKey); + + if (result != null && result.getObjectValue() != null) { + list = (List) result.getObjectValue(); + cache.remove(tripKey); + } else { + list = new ArrayList(); + } + + list.add(arrivalDeparture); + + Element arrivalDepartures = new Element(tripKey, Collections.synchronizedList(list)); + + cache.put(arrivalDepartures); } - - list.add(arrivalDeparture); - - Element arrivalDepartures = new Element(tripKey, Collections.synchronizedList(list)); - - cache.put(arrivalDepartures); + } + else + { + logger.error("Cannot add event to TripDataHistoryCache as it has no freqStartTime set. {}", arrivalDeparture); } - - } return tripKey; } diff --git a/transitclock/src/main/java/org/transitclock/core/dataCache/jcs/frequency/DwellTimeModelCache.java b/transitclock/src/main/java/org/transitclock/core/dataCache/jcs/frequency/DwellTimeModelCache.java index de10fc1e4..b26f1db83 100644 --- a/transitclock/src/main/java/org/transitclock/core/dataCache/jcs/frequency/DwellTimeModelCache.java +++ b/transitclock/src/main/java/org/transitclock/core/dataCache/jcs/frequency/DwellTimeModelCache.java @@ -141,7 +141,7 @@ private ArrivalDeparture findPreviousArrival(List stopData, Ar { if(!event.getVehicleId().equals(arrival.getVehicleId())) { - if(!event.getTripId().equals(arrival.getTripId())) + if(!event.getTripId().equals(arrival.getTripId())) { if(event.getStopId().equals(arrival.getStopId())) { @@ -149,6 +149,22 @@ private ArrivalDeparture findPreviousArrival(List stopData, Ar return event; } } + // If trip id the same check that not the same trip by checking the freqStartTime + if(event.getTripId().equals(arrival.getTripId())) + { + if(event.getFreqStartTime()!=null && arrival.getFreqStartTime()!=null) + { + if(!event.getFreqStartTime().equals(arrival.getFreqStartTime())) + { + if(event.getStopId().equals(arrival.getStopId())) + { + if(event.getTime() stopData, ArrivalDep return null; } @Override - public Long predictDwellTime(StopPathCacheKey cacheKey, Headway headway) { - //TODO null should be the start time of the freq bucket. - Integer time=FrequencyBasedHistoricalAverageCache.secondsFromMidnight(null ,2); - - time=FrequencyBasedHistoricalAverageCache.round(time, FrequencyBasedHistoricalAverageCache.getCacheIncrementsForFrequencyService()); + public Long predictDwellTime(StopPathCacheKey cacheKey, Headway headway) { TransitClockRLS rls=cache.get(cacheKey); if(rls!=null&&rls.getRls()!=null) From ea050e2dac25855584f40e7a5af6d157599abbfa Mon Sep 17 00:00:00 2001 From: scrudden Date: Thu, 13 Sep 2018 19:25:13 +0100 Subject: [PATCH 09/24] Tighten up filter values and discard dwell model if return silly values. --- .../dataCache/jcs/frequency/DwellTimeModelCache.java | 10 +++++++++- .../frequency/KalmanPredictionGeneratorImpl.java | 4 ++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/transitclock/src/main/java/org/transitclock/core/dataCache/jcs/frequency/DwellTimeModelCache.java b/transitclock/src/main/java/org/transitclock/core/dataCache/jcs/frequency/DwellTimeModelCache.java index b26f1db83..f7c5a637e 100644 --- a/transitclock/src/main/java/org/transitclock/core/dataCache/jcs/frequency/DwellTimeModelCache.java +++ b/transitclock/src/main/java/org/transitclock/core/dataCache/jcs/frequency/DwellTimeModelCache.java @@ -200,7 +200,15 @@ public Long predictDwellTime(StopPathCacheKey cacheKey, Headway headway) { double[] arg0 = new double[1]; arg0[0]=headway.getHeadway(); rls.getRls().predict(arg0); - return (long) Math.pow(10, rls.getRls().predict(arg0)); + long prediction = (long) Math.pow(10, rls.getRls().predict(arg0)); + + // If silly values returned then need to reset model and allow it use the super prediction. + if(prediction>maxDwellTimeAllowedInModel.getValue()) + { + cache.put(cacheKey,null); + return null; + } + return prediction; }else { return null; diff --git a/transitclock/src/main/java/org/transitclock/core/predictiongenerator/kalman/frequency/KalmanPredictionGeneratorImpl.java b/transitclock/src/main/java/org/transitclock/core/predictiongenerator/kalman/frequency/KalmanPredictionGeneratorImpl.java index 8d62395eb..1d6fa566c 100755 --- a/transitclock/src/main/java/org/transitclock/core/predictiongenerator/kalman/frequency/KalmanPredictionGeneratorImpl.java +++ b/transitclock/src/main/java/org/transitclock/core/predictiongenerator/kalman/frequency/KalmanPredictionGeneratorImpl.java @@ -47,11 +47,11 @@ public class KalmanPredictionGeneratorImpl extends HistoricalAveragePredictionGe * historical value. */ private static final IntegerConfigValue minKalmanDays = new IntegerConfigValue( - "transitclock.prediction.data.kalman.mindays", new Integer(2), + "transitclock.prediction.data.kalman.mindays", new Integer(3), "Min number of days trip data that needs to be available before Kalman prediciton is used instead of default transiTime prediction."); private static final IntegerConfigValue maxKalmanDays = new IntegerConfigValue( - "transitclock.prediction.data.kalman.maxdays", new Integer(2), + "transitclock.prediction.data.kalman.maxdays", new Integer(3), "Max number of historical days trips to include in Kalman prediction calculation."); private static final IntegerConfigValue maxKalmanDaysToSearch = new IntegerConfigValue( From a5d078335a25b4c771554f35963f36b664bbdf3b Mon Sep 17 00:00:00 2001 From: vperez Date: Thu, 13 Sep 2018 22:58:40 -0300 Subject: [PATCH 10/24] Add square view for loop routes - it requires to have only 1 direction (0) - if direction is not defined in vehicleDetail information, it will assume 0 --- .../org/transitclock/api/data/ApiShape.java | 13 + .../src/main/webapp/synoptic/index.jsp | 9 +- .../webapp/synoptic/javascript/synoptic.js | 369 +++++++++++++++--- 3 files changed, 333 insertions(+), 58 deletions(-) diff --git a/transitclockApi/src/main/java/org/transitclock/api/data/ApiShape.java b/transitclockApi/src/main/java/org/transitclock/api/data/ApiShape.java index be56ed114..8d4da484d 100644 --- a/transitclockApi/src/main/java/org/transitclock/api/data/ApiShape.java +++ b/transitclockApi/src/main/java/org/transitclock/api/data/ApiShape.java @@ -25,6 +25,7 @@ import org.transitclock.db.structs.Location; import org.transitclock.ipc.data.IpcShape; +import org.transitclock.utils.Geo; /** * A portion of a shape that defines a trip pattern. A List of ApiLocation @@ -52,7 +53,14 @@ public class ApiShape { private double length; @XmlAttribute private String directionId; + + //To define what kind of pattern is: circular (loop, one ending), linear (normal line with two different endings) + @XmlAttribute + private String patternType="linear"; + private static final int LOOP_ENDING_MAX_DISTANCE=150; + private static final String LOOP_PATTERN="circular"; + private static final String LINAR_PATTER="linear"; /********************** Member Functions **************************/ /** @@ -75,6 +83,11 @@ public ApiShape(IpcShape shape) { for (Location loc : shape.getLocations()) { this.locations.add(new ApiLocation(loc.getLat(), loc.getLon())); } + int size=shape.getLocations().size(); + if(size>0 && Geo.distance(shape.getLocations().get(0), shape.getLocations().get(size-1)) + + +
+ style="width: 100%; height: 350px; left: 0px; top: 50px; position: absolute;' ">