Skip to content

Commit

Permalink
Merge branch 'dev-flex' into flex-merge-feeds
Browse files Browse the repository at this point in the history
  • Loading branch information
@br648
br648 committed Nov 14, 2023
2 parents 3c6a6aa + e832925 commit dce4d70
Show file tree
Hide file tree
Showing 29 changed files with 608 additions and 202 deletions.
4 changes: 2 additions & 2 deletions src/main/java/com/conveyal/gtfs/loader/Field.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.SQLType;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.Set;

/**
Expand Down Expand Up @@ -47,7 +47,7 @@ public abstract class Field {
* Indicates that this field acts as a foreign key to this referenced table. This is used when checking referential
* integrity when loading a feed.
* */
public HashSet<Table> referenceTables = new HashSet<>();
public Set<Table> referenceTables = new LinkedHashSet<>();
private boolean shouldBeIndexed;
private boolean emptyValuePermitted;
private boolean isConditionallyRequired;
Expand Down
13 changes: 13 additions & 0 deletions src/main/java/com/conveyal/gtfs/loader/JDBCTableReader.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.conveyal.gtfs.loader;

import com.conveyal.gtfs.model.Calendar;
import com.conveyal.gtfs.model.Entity;
import com.conveyal.gtfs.storage.StorageException;
import gnu.trove.map.TObjectIntMap;
Expand Down Expand Up @@ -146,6 +147,18 @@ public int getRowCount() {
}
}

/**
* Provide reader for calendar table.
*/
public static JDBCTableReader<Calendar> getCalendarTableReader(DataSource dataSource, String tablePrefix) {
return new JDBCTableReader(
Table.CALENDAR,
dataSource,
tablePrefix + ".",
EntityPopulator.CALENDAR
);
}

private class EntityIterator implements Iterator<T> {

private Connection connection; // Will remain open for the duration of the iteration.
Expand Down
86 changes: 53 additions & 33 deletions src/main/java/com/conveyal/gtfs/loader/JdbcGtfsExporter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,16 @@
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.sql.Connection;
import java.sql.SQLException;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
Expand Down Expand Up @@ -86,7 +89,7 @@ public JdbcGtfsExporter(String feedId, String outFile, DataSource dataSource, bo
/**
* Utility method to check if an exception uses a specific service.
*/
public Boolean exceptionInvolvesService(ScheduleException ex, String serviceId) {
public boolean exceptionInvolvesService(ScheduleException ex, String serviceId) {
return (
ex.addedService.contains(serviceId) ||
ex.removedService.contains(serviceId) ||
Expand All @@ -106,7 +109,7 @@ public FeedLoadResult exportTables() {
FeedLoadResult result = new FeedLoadResult();

try {
zipOutputStream = new ZipOutputStream(new FileOutputStream(outFile));
zipOutputStream = new ZipOutputStream(Files.newOutputStream(Paths.get(outFile)));
long startTime = System.currentTimeMillis();
// We get a single connection object and share it across several different methods.
// This ensures that actions taken in one method are visible to all subsequent SQL statements.
Expand Down Expand Up @@ -142,40 +145,55 @@ public FeedLoadResult exportTables() {
if (fromEditor) {
// Export schedule exceptions in place of calendar dates if exporting a feed/schema that represents an editor snapshot.
GTFSFeed feed = new GTFSFeed();
// FIXME: The below table readers should probably just share a connection with the exporter.
JDBCTableReader<ScheduleException> exceptionsReader =
new JDBCTableReader(Table.SCHEDULE_EXCEPTIONS, dataSource, feedIdToExport + ".",
EntityPopulator.SCHEDULE_EXCEPTION);
JDBCTableReader<Calendar> calendarsReader =
new JDBCTableReader(Table.CALENDAR, dataSource, feedIdToExport + ".",
EntityPopulator.CALENDAR);
Iterable<Calendar> calendars = calendarsReader.getAll();
JDBCTableReader<ScheduleException> exceptionsReader =new JDBCTableReader(
Table.SCHEDULE_EXCEPTIONS,
dataSource,
feedIdToExport + ".",
EntityPopulator.SCHEDULE_EXCEPTION
);
JDBCTableReader<Calendar> calendarReader = JDBCTableReader.getCalendarTableReader(dataSource, feedIdToExport);
Iterable<Calendar> calendars = calendarReader.getAll();
Iterable<ScheduleException> exceptionsIterator = exceptionsReader.getAll();
List<ScheduleException> exceptions = new ArrayList<>();
// FIXME: Doing this causes the connection to stay open, but it is closed in the finalizer so it should
// not be a big problem.
for (ScheduleException exception : exceptionsIterator) {
exceptions.add(exception);
List<ScheduleException> calendarExceptions = new ArrayList<>();
List<ScheduleException> calendarDateExceptions = new ArrayList<>();
// Separate distinct calendar date exceptions from those associated with calendars.
for (ScheduleException ex : exceptionsIterator) {
if (ex.exemplar.equals(ScheduleException.ExemplarServiceDescriptor.CALENDAR_DATE_SERVICE)) {
calendarDateExceptions.add(ex);
} else {
calendarExceptions.add(ex);
}
}

int calendarDateCount = calendarDateExceptions.size();
// Extract calendar date services, convert to calendar date and add to the feed.
for (ScheduleException ex : calendarDateExceptions) {
for (LocalDate date : ex.dates) {
String serviceId = ex.customSchedule.get(0);
CalendarDate calendarDate = new CalendarDate();
calendarDate.date = date;
calendarDate.service_id = serviceId;
calendarDate.exception_type = 1;
Service service = new Service(serviceId);
service.calendar_dates.put(date, calendarDate);
// If the calendar dates provided contain duplicates (e.g. two or more identical service ids
// that are NOT associated with a calendar) only the first entry would persist export. To
// resolve this a unique key consisting of service id and date is used.
feed.services.put(String.format("%s-%s", calendarDate.service_id, calendarDate.date), service);
}
}

// check whether the feed is organized in a format with the calendars.txt file
if (calendarsReader.getRowCount() > 0) {
if (calendarReader.getRowCount() > 0) {
// feed does have calendars.txt file, continue export with strategy of matching exceptions
// to calendar to output calendar_dates.txt
int calendarDateCount = 0;
for (Calendar cal : calendars) {
Service service = new Service(cal.service_id);
service.calendar = cal;
for (ScheduleException ex : exceptions.stream()
for (ScheduleException ex : calendarExceptions.stream()
.filter(ex -> exceptionInvolvesService(ex, cal.service_id))
.collect(Collectors.toList())
) {
if (ex.exemplar.equals(ScheduleException.ExemplarServiceDescriptor.SWAP) &&
(!ex.addedService.contains(cal.service_id) && !ex.removedService.contains(cal.service_id))) {
// Skip swap exception if cal is not referenced by added or removed service.
// This is not technically necessary, but the output is cleaner/more intelligible.
continue;
}

for (LocalDate date : ex.dates) {
if (date.isBefore(cal.start_date) || date.isAfter(cal.end_date)) {
// No need to write dates that do not apply
Expand All @@ -189,22 +207,24 @@ public FeedLoadResult exportTables() {
LOG.info("Adding exception {} (type={}) for calendar {} on date {}", ex.name, calendarDate.exception_type, cal.service_id, date);

if (service.calendar_dates.containsKey(date))
throw new IllegalArgumentException("Duplicate schedule exceptions on " + date.toString());
throw new IllegalArgumentException("Duplicate schedule exceptions on " + date);

service.calendar_dates.put(date, calendarDate);
calendarDateCount += 1;
}
}
feed.services.put(cal.service_id, service);
}
if (calendarDateCount == 0) {
LOG.info("No calendar dates found. Skipping table.");
} else {
LOG.info("Writing {} calendar dates from schedule exceptions", calendarDateCount);
new CalendarDate.Writer(feed).writeTable(zipOutputStream);
}
}
if (calendarDateCount == 0) {
LOG.info("No calendar dates found. Skipping table.");
} else {
// No calendar records exist, export calendar_dates as is and hope for the best.
LOG.info("Writing {} calendar dates from schedule exceptions", calendarDateCount);
new CalendarDate.Writer(feed).writeTable(zipOutputStream);
}

if (calendarReader.getRowCount() == 0 && calendarDateExceptions.isEmpty()) {
// No calendar or calendar date service records exist, export calendar_dates as is and hope for the best.
// This situation will occur in at least 2 scenarios:
// 1. A GTFS has been loaded into the editor that had only the calendar_dates.txt file
// and no further edits were made before exporting to a snapshot
Expand Down
Loading

0 comments on commit dce4d70

Please sign in to comment.