Skip to content

JSON > + service_id_avoid_merge... #76

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mmathieum merged 3 commits into from
Jul 7, 2026
Merged

JSON > + service_id_avoid_merge... #76

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1d2f132
`JSON` > + `service_id_avoid_merge`...
mmathieum Jul 7, 2026
28cbd71
PR comments
mmathieum Jul 7, 2026
161d6c7
fix
mmathieum Jul 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
167 changes: 109 additions & 58 deletions src/main/java/org/mtransit/parser/DefaultAgencyTools.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ public class DefaultAgencyTools implements GAgencyTools {
// 2024-04-09: 10x -> 3x because merging 2 schedule can create very bad schedule info #GRTbus
private static final long MAX_CALENDAR_DATE_COVERAGE_RATIO = 3;

private static final int MIN_PREVIOUS_NEXT_ADDED_DAYS = 2; // = 1 day
private static final int MIN_PREVIOUS_NEXT_ADDED_DAYS = 1;

@SuppressWarnings("WeakerAccess")
public static final boolean GOOD_ENOUGH_ACCEPTED;
Expand Down Expand Up @@ -457,6 +457,41 @@ public String cleanServiceId(@NotNull String gServiceId, boolean keep) {
return cleanServiceId;
}

/**
* When using only `calendar_dates.txt`, we might want to avoid adding 1 more service day if it adds a whole new "service ID period".
* For example, we can have many service IDs starting with YYMMDD-abc-123 with 2 distinct groups like 26JN29-... and 26AU03-....
*/
private static boolean avoidServiceIdMerge(@NotNull Collection<Integer> currentServiceIdInts, @NotNull Collection<Integer> newServiceIdInts) {
final boolean avoidServiceIdMerge = Configs.getAgencyConfig() != null && Configs.getAgencyConfig().getServiceIdAvoidMerge();
if (!avoidServiceIdMerge) return true; // can merge
final boolean serviceIdCleanMerged = Configs.getAgencyConfig() != null && Configs.getAgencyConfig().getServiceIdCleanMerged();
final String serviceIdCleanupRegex = Configs.getAgencyConfig() == null ? null : Configs.getAgencyConfig().getServiceIdCleanupRegex();
if (!serviceIdCleanMerged && (serviceIdCleanupRegex == null || serviceIdCleanupRegex.isBlank())) return true; // can merge
final Pattern serviceIdCleanupPattern = serviceIdCleanupRegex == null ? null : Pattern.compile(serviceIdCleanupRegex);
Comment thread
mmathieum marked this conversation as resolved.
Outdated
for (Integer newServiceIdInt : newServiceIdInts) {
String newServiceId = GIDs.getString(newServiceIdInt);
if (serviceIdCleanupPattern != null) {
newServiceId = GTFSCommons.cleanOriginalString(newServiceId, serviceIdCleanupPattern);
}
if (serviceIdCleanMerged) {
newServiceId = CleanUtils.cleanMergedID(newServiceId);
}
for (Integer currentServiceIdInt : currentServiceIdInts) {
String currentServiceId = GIDs.getString(currentServiceIdInt);
if (serviceIdCleanupPattern != null) {
currentServiceId = GTFSCommons.cleanOriginalString(currentServiceId, serviceIdCleanupPattern);
}
if (serviceIdCleanMerged) {
currentServiceId = CleanUtils.cleanMergedID(currentServiceId);
}
if (newServiceId.equals(currentServiceId)) {
return false; // can not merge
}
}
}
return true; // can merge
}
Comment thread
mmathieum marked this conversation as resolved.
Outdated

@Nullable
@Override
public String getServiceIdCleanupRegex() {
Expand Down Expand Up @@ -964,6 +999,7 @@ public boolean removeRouteDescFromDirectionHeadsign() {
directionHeadSign = "";
}
}
//noinspection deprecation
return cleanDirectionHeadsign(gRoute, directionId, fromStopName, directionHeadSign);
}

Expand Down Expand Up @@ -1515,9 +1551,7 @@ public int getThreadPoolSize() {

@Nullable
@Override
public Pair<Integer, Integer> getTimes(@NotNull GStopTime gStopTime,
@NotNull List<GStopTime> tripStopTimes,
@NotNull DateFormat timeFormat) {
public Pair<Integer, Integer> getTimes(@NotNull GStopTime gStopTime, @NotNull List<GStopTime> tripStopTimes, @NotNull DateFormat timeFormat) {
if (gStopTime.hasArrivalTime() && gStopTime.hasDepartureTime()) {
return new Pair<>(
TimeUtils.cleanExtraSeconds(gStopTime.getArrivalTime()),
Expand All @@ -1532,9 +1566,7 @@ public boolean allowIgnoreInvalidStopTimes() {
}

@Nullable
private static Pair<Integer, Integer> extractTimes(GStopTime gStopTime,
@NotNull List<GStopTime> tripStopTimes,
DateFormat timeFormat) {
private static Pair<Integer, Integer> extractTimes(GStopTime gStopTime, @NotNull List<GStopTime> tripStopTimes, DateFormat timeFormat) {
try {
Pair<Long, Long> timesInMs = extractTimeInMs(gStopTime, tripStopTimes);
if (timesInMs == null) return null;
Expand All @@ -1561,8 +1593,7 @@ private static Pair<Integer, Integer> extractTimes(GStopTime gStopTime,
}

@Nullable
public static Pair<Long, Long> extractTimeInMs(@NotNull GStopTime gStopTime,
@NotNull List<GStopTime> tripGStopTimes) {
public static Pair<Long, Long> extractTimeInMs(@NotNull GStopTime gStopTime, @NotNull List<GStopTime> tripGStopTimes) {
int previousArrivalTime = -1;
int previousArrivalTimeStopSequence = -1;
int previousDepartureTime = -1;
Expand Down Expand Up @@ -1815,48 +1846,52 @@ static void parseCalendarDates(
boolean lookBackward,
@Nullable List<MServiceDate> lastServiceDates
) {
HashSet<Integer> todayServiceIds = findCalendarDatesTodayServiceIds(gCalendarDates, c, p, lookBackward, 1, true);
if (todayServiceIds.isEmpty()) {
final HashSet<Integer> todayServiceIdInts = findCalendarDatesPeriodServiceIdInts(gCalendarDates, c, p, lookBackward, 1, true);
if (todayServiceIdInts.isEmpty()) {
MTLog.log("NO schedule available for %s in calendar dates.", p.getTodayStringInt());
return;
}
refreshStartEndDatesFromCalendarDates(p, todayServiceIds, gCalendarDates);
refreshPeriodStartEndDatesFromCalendarDates(p, todayServiceIdInts, gCalendarDates);
while (true) {
MTLog.log("> Schedules from %s to %s... ", p.getStartDate(), p.getEndDate());
for (GCalendarDate gCalendarDate : gCalendarDates) {
if (gCalendarDate.isBetween(p.getStartDate(), p.getEndDate())
&& !gCalendarDate.isServiceIdInts(todayServiceIds)) {
&& !gCalendarDate.isServiceIdInts(todayServiceIdInts)) {
//noinspection DiscouragedApi
MTLog.log("> new service ID from calendar date active on %s between %s and %s: '%s'", gCalendarDate.getDate(), p.getStartDate(), p.getEndDate(), gCalendarDate.getServiceId());
todayServiceIds.add(gCalendarDate.getServiceIdInt());
todayServiceIdInts.add(gCalendarDate.getServiceIdInt());
}
}
boolean newDates = refreshStartEndDatesFromCalendarDates(p, todayServiceIds, gCalendarDates);
final boolean newDates = refreshPeriodStartEndDatesFromCalendarDates(p, todayServiceIdInts, gCalendarDates);
if (newDates) {
MTLog.log("> new start date '%s' & end date '%s' from calendar date active during service ID(s).", p.getStartDate(), p.getEndDate());
continue;
}
Period pNext = new Period();
pNext.setTodayStringInt(incDateDays(DATE_FORMAT, c, p.getEndDate(), 1));
HashSet<Integer> nextDayServiceIds = findCalendarDatesTodayServiceIds(gCalendarDates, c, pNext, false, 0, false);
refreshStartEndDatesFromCalendarDates(pNext, nextDayServiceIds, gCalendarDates);
if (pNext.getStartDate() != null && pNext.getEndDate() != null
&& diffLowerThan(DATE_FORMAT, c, pNext.getStartDate(), pNext.getEndDate(), MIN_PREVIOUS_NEXT_ADDED_DAYS)) {
p.setEndDate(pNext.getEndDate());
MTLog.log("> new end date '%s' because next %d day has own service ID(s)", p.getEndDate(), MIN_PREVIOUS_NEXT_ADDED_DAYS);
final Period nextDayPeriod = Period.from(incDateDays(DATE_FORMAT, c, p.getEndDate(), 1));
final HashSet<Integer> nextDayServiceIdInts = findCalendarDatesPeriodServiceIdInts(gCalendarDates, c, nextDayPeriod, false, 0, false);
refreshPeriodStartEndDatesFromCalendarDates(nextDayPeriod, nextDayServiceIdInts, gCalendarDates);
if (nextDayPeriod.getStartDate() != null && nextDayPeriod.getEndDate() != null
&& diffLowerOrEqual(DATE_FORMAT, c, nextDayPeriod.getStartDate(), nextDayPeriod.getEndDate(), MIN_PREVIOUS_NEXT_ADDED_DAYS)
&& avoidServiceIdMerge(todayServiceIdInts, nextDayServiceIdInts)) {
Comment thread
mmathieum marked this conversation as resolved.
Outdated
p.setEndDate(nextDayPeriod.getEndDate());
MTLog.log("> new end date '%s' because next %d day(s) has own service ID(s)", p.getEndDate(), MIN_PREVIOUS_NEXT_ADDED_DAYS);
continue;
}
Period pPrevious = new Period();
pPrevious.setTodayStringInt(incDateDays(DATE_FORMAT, c, p.getStartDate(), -1));
HashSet<Integer> previousDayServiceIds = findCalendarDatesTodayServiceIds(gCalendarDates, c, pPrevious, false, 0, false);
refreshStartEndDatesFromCalendarDates(pPrevious, previousDayServiceIds, gCalendarDates);
final Period previousDayPeriod = Period.from(incDateDays(DATE_FORMAT, c, p.getStartDate(), -1));
final HashSet<Integer> previousDayPeriodServiceIdInts = findCalendarDatesPeriodServiceIdInts(gCalendarDates, c, previousDayPeriod, false, 0, false);
refreshPeriodStartEndDatesFromCalendarDates(
previousDayPeriod,
previousDayPeriodServiceIdInts,
gCalendarDates
);
if (lookBackward // NOT next schedule, only current schedule can look behind
&& pPrevious.getStartDate() != null && pPrevious.getEndDate() != null
&& diffLowerThan(DATE_FORMAT, c, pPrevious.getStartDate(), pPrevious.getEndDate(), MIN_PREVIOUS_NEXT_ADDED_DAYS)
&& MServiceDate.containsAllServiceIdInts(lastServiceDates, previousDayServiceIds) // 2026-01-29: only look backward if min calendar date coverage is not reached or in previous schedule
&& previousDayPeriod.getStartDate() != null && previousDayPeriod.getEndDate() != null
&& diffLowerOrEqual(DATE_FORMAT, c, previousDayPeriod.getStartDate(), previousDayPeriod.getEndDate(), MIN_PREVIOUS_NEXT_ADDED_DAYS)
&& MServiceDate.containsAllServiceIdInts(lastServiceDates, previousDayPeriodServiceIdInts) // 2026-01-29: only look backward if min calendar date coverage is not reached or in previous schedule
&& avoidServiceIdMerge(todayServiceIdInts, previousDayPeriodServiceIdInts)
) {
Comment thread
mmathieum marked this conversation as resolved.
p.setStartDate(pPrevious.getStartDate());
MTLog.log("> new start date '%s' because previous %d day has own service ID(s)", p.getStartDate(), MIN_PREVIOUS_NEXT_ADDED_DAYS);
p.setStartDate(previousDayPeriod.getStartDate());
MTLog.log("> new start date '%s' because previous %d day(s) has own service ID(s)", p.getStartDate(), MIN_PREVIOUS_NEXT_ADDED_DAYS);
continue;
}
if (diffLowerThan(DATE_FORMAT, c, p.getStartDate(), p.getEndDate(), MIN_CALENDAR_DATE_COVERAGE_TOTAL_IN_DAYS)) {
Expand All @@ -1865,15 +1900,15 @@ && diffLowerThan(DATE_FORMAT, c, pPrevious.getStartDate(), pPrevious.getEndDate(
p.getStartDate(), // morning
p.getEndDate() + 1 // midnight ≈ tomorrow
);
long nextPeriodCoverageInMs = pNext.getStartDate() == null || pNext.getEndDate() == null ? 0L :
long nextPeriodCoverageInMs = nextDayPeriod.getStartDate() == null || nextDayPeriod.getEndDate() == null ? 0L :
diffInMs(DATE_FORMAT, c,
pNext.getStartDate(), // morning
pNext.getEndDate() + 1 // midnight ≈ tomorrow
nextDayPeriod.getStartDate(), // morning
nextDayPeriod.getEndDate() + 1 // midnight ≈ tomorrow
);
long previousPeriodCoverageInMs = pPrevious.getStartDate() == null || pPrevious.getEndDate() == null ? 0L :
long previousPeriodCoverageInMs = previousDayPeriod.getStartDate() == null || previousDayPeriod.getEndDate() == null ? 0L :
diffInMs(DATE_FORMAT, c,
pPrevious.getStartDate(), // morning
pPrevious.getEndDate() + 1 // midnight ≈ tomorrow
previousDayPeriod.getStartDate(), // morning
previousDayPeriod.getEndDate() + 1 // midnight ≈ tomorrow
);
long previousToCurrent = previousPeriodCoverageInMs / currentPeriodCoverageInMs;
long nextToCurrent = nextPeriodCoverageInMs / currentPeriodCoverageInMs;
Expand All @@ -1897,7 +1932,7 @@ && diffLowerThan(DATE_FORMAT, c, pPrevious.getStartDate(), pPrevious.getEndDate(
}
}

private static boolean refreshStartEndDatesFromCalendarDates(
private static boolean refreshPeriodStartEndDatesFromCalendarDates(
@NotNull Period p,
@NotNull Collection<Integer> serviceIds,
@NotNull List<GCalendarDate> gCalendarDates
Expand All @@ -1918,12 +1953,14 @@ private static boolean refreshStartEndDatesFromCalendarDates(
}

@NotNull
private static HashSet<Integer> findCalendarDatesTodayServiceIds(@NotNull List<GCalendarDate> gCalendarDates,
Calendar c,
Period p,
boolean lookBackward,
int minTodayServiceIdsSize,
boolean canIncDays) {
private static HashSet<Integer> findCalendarDatesPeriodServiceIdInts(
@NotNull List<GCalendarDate> gCalendarDates,
Calendar c,
Period p,
boolean lookBackward,
int minTodayServiceIdsSize,
boolean canIncDays
) {
final HashSet<Integer> todayServiceIds = new HashSet<>();
final Integer initialTodayStringInt = p.getTodayStringInt();
while (true) {
Expand Down Expand Up @@ -1990,9 +2027,10 @@ static void parseCalendars(@NotNull List<GCalendar> gCalendars, @Nullable List<G
pNext.setTodayStringInt(incDateDays(DATE_FORMAT, c, p.getEndDate(), 1));
findDayServiceIdsPeriod(gCalendars, gCalendarDates, pNext);
if (pNext.getStartDate() != null && pNext.getEndDate() != null
&& diffLowerThan(DATE_FORMAT, c, pNext.getStartDate(), pNext.getEndDate(), MIN_PREVIOUS_NEXT_ADDED_DAYS)) {
&& diffLowerOrEqual(DATE_FORMAT, c, pNext.getStartDate(), pNext.getEndDate(), MIN_PREVIOUS_NEXT_ADDED_DAYS)) {
// TODO use avoidServiceIdMerge()?
p.setEndDate(pNext.getEndDate());
MTLog.log("[parse-calendars] > new end date '%s' because next day has own service ID(s)", p.getEndDate());
MTLog.log("[parse-calendars] > new end date '%s' because next day(s) has own service ID(s)", p.getEndDate());
continue;
} else if (pNext.getStartDate() != null && pNext.getEndDate() != null) {
MTLog.logDebug("[parse-calendars] > ignore next period because coverage '%s' not < %s days: %s",
Expand Down Expand Up @@ -2211,10 +2249,7 @@ private static Period getEntirePeriodMinMaxDate(List<GCalendar> gCalendars, List
return p;
}

public static int incDateDays(@NotNull DateFormat dateFormat,
@NotNull Calendar calendar,
@Nullable Integer dateInt,
int numberOfDays) {
public static int incDateDays(@NotNull DateFormat dateFormat, @NotNull Calendar calendar, @Nullable Integer dateInt, int numberOfDays) {
try {
calendar.setTime(dateFormat.parse(String.valueOf(dateInt)));
calendar.add(Calendar.DAY_OF_MONTH, numberOfDays);
Expand All @@ -2225,11 +2260,28 @@ public static int incDateDays(@NotNull DateFormat dateFormat,
}

@SuppressWarnings("WeakerAccess")
public static boolean diffLowerThan(@NotNull DateFormat dateFormat,
@NotNull Calendar calendar,
@Nullable Integer startDateInt,
@Nullable Integer endDateInt,
int diffInDays) {
public static boolean diffLowerOrEqual(
@NotNull DateFormat dateFormat,
@NotNull Calendar calendar,
@Nullable Integer startDateInt,
@Nullable Integer endDateInt,
int diffInDays
) {
try {
return diffInMs(dateFormat, calendar, startDateInt, endDateInt) <= TimeUnit.DAYS.toMillis(diffInDays);
} catch (Exception e) {
throw new MTLog.Fatal(e, "Error while checking date difference!");
}
}

@SuppressWarnings("WeakerAccess")
public static boolean diffLowerThan(
@NotNull DateFormat dateFormat,
@NotNull Calendar calendar,
@Nullable Integer startDateInt,
@Nullable Integer endDateInt,
int diffInDays
) {
try {
return diffInMs(dateFormat, calendar, startDateInt, endDateInt) < TimeUnit.DAYS.toMillis(diffInDays);
} catch (Exception e) {
Expand Down Expand Up @@ -2289,8 +2341,7 @@ protected static boolean excludeUselessCalendarDateInt(@NotNull GCalendarDate gC
}

@SuppressWarnings("WeakerAccess")
protected static boolean excludeUselessTripInt(@NotNull GTrip gTrip,
@Nullable HashSet<Integer> serviceIds) {
protected static boolean excludeUselessTripInt(@NotNull GTrip gTrip, @Nullable HashSet<Integer> serviceIds) {
if (serviceIds != null) {
boolean knownServiceId = gTrip.isServiceIdInts(serviceIds);
//noinspection RedundantIfStatement
Expand Down
17 changes: 12 additions & 5 deletions src/main/java/org/mtransit/parser/Period.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,16 @@
package org.mtransit.parser

data class Period(
var todayStringInt: Int?,
var startDate: Int?,
var endDate: Int?,
data class Period @JvmOverloads constructor(
var todayStringInt: Int? = null,
var startDate: Int? = null,
var endDate: Int? = null,
) {
constructor() : this(null, null, null)
companion object {
@JvmStatic
fun from(today: Int) = Period(
todayStringInt = today,
startDate = null,
endDate = null
)
}
}
2 changes: 2 additions & 0 deletions src/main/java/org/mtransit/parser/config/gtfs/data/AgencyConfig.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,8 @@ data class AgencyConfig(
val serviceIdCleanMerged: Boolean = false, // OPT-IN feature
@SerialName("service_id_not_unique_allowed")
val serviceIdNotUniqueAllowed: Boolean = false, // OPT-IN feature
@SerialName("service_id_avoid_merge")
val serviceIdAvoidMerge: Boolean = false, // OPT-IN feature
) {
fun getAllLanguages(supportedLanguages: List<Locale>): List<Locale>? {
if (supportedLanguages.isEmpty()) return null
Expand Down
4 changes: 1 addition & 3 deletions src/main/java/org/mtransit/parser/gtfs/GAgencyTools.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -353,9 +353,7 @@ public interface GAgencyTools {

// SCHEDULE
@Nullable
Pair<Integer, Integer> getTimes(@NotNull GStopTime gStopTime,
@NotNull List<GStopTime> tripStopTimes,
@NotNull DateFormat timeFormat);
Pair<Integer, Integer> getTimes(@NotNull GStopTime gStopTime, @NotNull List<GStopTime> tripStopTimes, @NotNull DateFormat timeFormat);

boolean allowIgnoreInvalidStopTimes();

Expand Down