Skip to content

Commit

Permalink
🧐 Fix remaining sync issues, adjust schema (#58)
Browse files Browse the repository at this point in the history 
This change resolves remaining issues introduced by #57 and #53.

The schema has had some slight adjustments made necessary by changes to
MyPurdue as well as breaking changes to the database library detailed
[here](https://www.roji.org/postgresql-dotnet-timestamp-mapping):

- `Capacity`, `Enrolled`, `RemainingSpace`, `WaitListCapacity`,
`WaitListCount`, and `WaitListSpace` are **no longer available** on
`Section`. Recent changes to MyPurdue have made it not feasible to
synchronize these values. This is tracked in #56 for a potential future
solution.
- `StartDate`/`EndDate` on `Meeting`, `Section`, and `Term` is now
stored as
[`DateOnly`](https://learn.microsoft.com/en-us/dotnet/api/system.dateonly)
objects, and may be null
- `StartTime` on `Meeting` is stored as a
[`TimeOnly`](https://learn.microsoft.com/en-us/dotnet/api/system.timeonly)
object, and may be null
- All dates and times are local time with no offset or time zone
information.

Some offline conversion tables were introduced for pieces of information
CatalogSync no longer has access to, such as campus/building short
codes. Tracked in #54 and #55 for potential future solutions.
  • Loading branch information
@haydenmc
haydenmc authored Nov 14, 2023
1 parent 11dcdb7 commit 2014d2d
Show file tree
Hide file tree
Showing 21 changed files with 260 additions and 1,065 deletions.
100 changes: 23 additions & 77 deletions src/CatalogSync/FastSync.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,8 +135,8 @@ private async Task InternalSynchronizeAsync(TermSyncBehavior termSyncBehavior,
Id = Guid.NewGuid(),
Code = scrapedTerm.Id,
Name = scrapedTerm.Name,
StartDate = DateTimeOffset.MinValue,
EndDate = DateTimeOffset.MaxValue,
StartDate = null,
EndDate = null,
};
dbContext.Add(dbTerm);
terms[scrapedTerm.Id] = dbTerm;
Expand All @@ -150,7 +150,9 @@ private async Task InternalSynchronizeAsync(TermSyncBehavior termSyncBehavior,
{
// We only care about terms that haven't ended yet -
// we still want to sync "future" terms.
if (terms[scrapedTerm.Id].EndDate > DateTimeOffset.Now)
if (terms[scrapedTerm.Id].EndDate == null ||
(terms[scrapedTerm.Id].EndDate >
DateOnly.FromDateTime(DateTime.Now.AddDays(-1))))
{
termsToSync.Add(terms[scrapedTerm.Id]);
}
Expand Down Expand Up @@ -247,28 +249,16 @@ private async Task InternalSynchronizeTermAsync(DatabaseTerm term,
{
dbContext.Entry(term).Property(t => t.StartDate).CurrentValue =
dbContext.Sections
.Where(s =>
(s.Class.TermId == term.Id) &&
(s.StartDate > DateTimeOffset.MinValue.AddDays(7))) // Add this buffer,
// since DTO on
// SQLite has some
// funky timezone
// limitations
.Where(s => (s.Class.TermId == term.Id) && (s.StartDate != null))
.Select(s => s.StartDate)
.OrderBy(d => d)
.ToList()
.DefaultIfEmpty(DateTimeOffset.MinValue)
.First();
.FirstOrDefault();
dbContext.Entry(term).Property(t => t.EndDate).CurrentValue =
dbContext.Sections
.Where(s =>
(s.Class.TermId == term.Id) &&
(s.EndDate < DateTimeOffset.MaxValue.AddDays(-7)))
.Where(s => (s.Class.TermId == term.Id) && (s.EndDate != null))
.Select(s => s.EndDate)
.OrderByDescending(d => d)
.ToList()
.DefaultIfEmpty(DateTimeOffset.MaxValue)
.First();
.FirstOrDefault();
dbContext.Entry(term).State = EntityState.Modified;
dbContext.SaveChanges();
}
Expand Down Expand Up @@ -470,15 +460,15 @@ private DatabaseSection AddOrUpdateSection(Guid classId,
ICollection<DatabaseSection> dbSections, ScrapedSection section)
{
var startDate = section.Meetings
.OrderBy(m => m.StartDate)
.Select(m => m.StartDate)
.DefaultIfEmpty(DateTimeOffset.MinValue)
.Where(d => d != null)
.OrderBy(d => d)
.FirstOrDefault();

var endDate = section.Meetings
.OrderByDescending(m => m.EndDate)
.Select(m => m.EndDate)
.DefaultIfEmpty(DateTimeOffset.MaxValue)
.Where(d => d != null)
.OrderByDescending(d => d)
.FirstOrDefault();

var existingSection = dbSections.SingleOrDefault(s => (s.Crn == section.Crn));
Expand Down Expand Up @@ -511,44 +501,7 @@ private DatabaseSection AddOrUpdateSection(Guid classId,
dbEntry.Property(s => s.EndDate).CurrentValue = endDate;
modified = true;
}
if (existingSection.Capacity != section.Capacity)
{
existingSection.Capacity = section.Capacity;
dbEntry.Property(s => s.Capacity).CurrentValue = section.Capacity;
modified = true;
}
if (existingSection.Enrolled != section.Enrolled)
{
existingSection.Enrolled = section.Enrolled;
dbEntry.Property(s => s.Enrolled).CurrentValue = section.Enrolled;
modified = true;
}
if (existingSection.RemainingSpace != section.RemainingSpace)
{
existingSection.RemainingSpace = section.RemainingSpace;
dbEntry.Property(s => s.RemainingSpace).CurrentValue =
section.RemainingSpace;
modified = true;
}
if (existingSection.WaitListCapacity != section.WaitListCapacity)
{
existingSection.WaitListCapacity = section.WaitListCapacity;
dbEntry.Property(s => s.WaitListCapacity).CurrentValue =
section.WaitListCapacity;
modified = true;
}
if (existingSection.WaitListCount != section.WaitListCount)
{
existingSection.WaitListCount = section.WaitListCount;
dbEntry.Property(s => s.WaitListCount).CurrentValue = section.WaitListCount;
modified = true;
}
if (existingSection.WaitListSpace != section.WaitListSpace)
{
existingSection.WaitListSpace = section.WaitListSpace;
dbEntry.Property(s => s.WaitListSpace).CurrentValue = section.WaitListSpace;
modified = true;
}

if (modified)
{
dbEntry.State = EntityState.Modified;
Expand All @@ -563,15 +516,8 @@ private DatabaseSection AddOrUpdateSection(Guid classId,
Crn = section.Crn,
ClassId = classId,
Type = section.Type,
RegistrationStatus = RegistrationStatus.NotAvailable,
StartDate = startDate,
EndDate = endDate,
Capacity = section.Capacity,
Enrolled = section.Enrolled,
RemainingSpace = section.RemainingSpace,
WaitListCapacity = section.WaitListCapacity,
WaitListCount = section.WaitListCount,
WaitListSpace = section.WaitListSpace,
};
dbContext.Add(newSection);
return newSection;
Expand All @@ -587,7 +533,11 @@ private void AddOrUpdateSectionMeetings(DatabaseCampus campus, DatabaseSection d
var existingMeetingsToKeep = new List<DatabaseMeeting>();
foreach (var meeting in section.Meetings)
{
var meetingDuration = meeting.EndTime.Subtract(meeting.StartTime);
var meetingDuration = TimeSpan.Zero;
if ((meeting.EndTime != null) && (meeting.StartTime != null))
{
meetingDuration = ((TimeOnly)meeting.EndTime - (TimeOnly)meeting.StartTime);
}
var dbRoom = FetchOrAddMeetingRoom(campus, meeting);
var dbInstructors = FetchOrAddMeetingInstructors(meeting);

Expand All @@ -598,14 +548,10 @@ private void AddOrUpdateSectionMeetings(DatabaseCampus campus, DatabaseSection d
var dbMeeting = dbMeetings.FirstOrDefault(m =>
(m.RoomId == dbRoom.Id) &&
(m.DaysOfWeek == (DatabaseDaysOfWeek) meeting.DaysOfWeek) &&
(m.StartDate.Subtract(meeting.StartDate).Duration()
<= MEETING_TIME_EQUALITY_TOLERANCE) &&
(m.EndDate.Subtract(meeting.EndDate).Duration()
<= MEETING_TIME_EQUALITY_TOLERANCE) &&
(m.StartTime.Subtract(meeting.StartTime).Duration()
<= MEETING_TIME_EQUALITY_TOLERANCE) &&
(m.Duration.Subtract(meetingDuration).Duration()
<= MEETING_TIME_EQUALITY_TOLERANCE));
(m.StartDate == meeting.StartDate) &&
(m.EndDate == meeting.EndDate) &&
(m.StartTime == meeting.StartTime) &&
(m.Duration == meetingDuration));

if (dbMeeting == null)
{
Expand Down
11 changes: 8 additions & 3 deletions src/CatalogSync/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,11 +34,11 @@ public class Options
HelpText = "Sync all terms, don't skip old/existing terms")]
public bool SyncAllTerms { get; set; }

[Option(shortName: 't', longName: "terms",
[Option(shortName: 't', longName: "terms", Separator = ',',
HelpText = "Term codes to sync (ex. 202210)")]
public IEnumerable<string> Terms { get; set; }

[Option(shortName: 's', longName: "subjects",
[Option(shortName: 's', longName: "subjects", Separator = ',',
HelpText = "Subject codes to sync (ex. CS)")]
public IEnumerable<string> Subjects { get; set; }
}
Expand Down Expand Up @@ -75,6 +75,7 @@ static async Task RunASync(Options options)
if (options.DataProvider == DataProvider.Sqlite)
{
var dbOptions = new DbContextOptionsBuilder<ApplicationDbContext>()
//.UseLoggerFactory(loggerFactory)
.UseSqlite(options.ConnectionString,
o => o.MigrationsAssembly("Database.Migrations.Sqlite"))
.Options;
Expand All @@ -83,8 +84,12 @@ static async Task RunASync(Options options)
else if (options.DataProvider == DataProvider.Npgsql)
{
var dbOptions = new DbContextOptionsBuilder<ApplicationDbContext>()
//.UseLoggerFactory(loggerFactory)
.UseNpgsql(options.ConnectionString,
o => o.MigrationsAssembly("Database.Migrations.Npgsql"))
o => o.MigrationsAssembly("Database.Migrations.Npgsql")
.MaxBatchSize(16)) // HACK: A batch size any larger than this
// results in intermittent connection
// problems on PostgreSQL/Linux
.Options;
dbContext = new ApplicationDbContext(dbOptions);
}
Expand Down
Loading

0 comments on commit 2014d2d

Please sign in to comment.