diff --git a/ClassTranscribeDatabase/Services/MSTranscription/MSTranscriptionService.cs b/ClassTranscribeDatabase/Services/MSTranscription/MSTranscriptionService.cs index 472aeb9..d42565b 100644 --- a/ClassTranscribeDatabase/Services/MSTranscription/MSTranscriptionService.cs +++ b/ClassTranscribeDatabase/Services/MSTranscription/MSTranscriptionService.cs @@ -191,8 +191,8 @@ private async Task performRecognitionAsync(string logId, string fileP if (verboseLogging) { - _logger.LogInformation($"{logId}: Begin={begin.Minutes}:{begin.Seconds},{begin.Milliseconds}", begin); - _logger.LogInformation($"{logId}: End={end.Minutes}:{end.Seconds},{end.Milliseconds}"); + _logger.LogInformation($"{logId}: Begin={begin.Minutes}:{begin.Seconds}.{begin.Milliseconds}", begin); + _logger.LogInformation($"{logId}: End={end.Minutes}:{end.Seconds}.{end.Milliseconds}"); } // TODO/TOREVIEW: // ToCaptionEntitiesWithWordTiming vs ToCaptionEntitiesInterpolate @@ -243,7 +243,8 @@ private async Task performRecognitionAsync(string logId, string fileP } else if (e.Result.Reason == ResultReason.NoMatch) { - _logger.LogInformation($"{logId}: NOMATCH: Speech could not be recognized."); + TimeSpan begin = (new TimeSpan(e.Result.OffsetInTicks)).Add(restartOffset); + _logger.LogInformation($"{logId}: NOMATCH: ({begin.Minutes}:{begin.Seconds}) Speech could not be recognized."); } }; diff --git a/ClassTranscribeServer/Controllers/AdminController.cs b/ClassTranscribeServer/Controllers/AdminController.cs index 1ad459a..f9a88a7 100644 --- a/ClassTranscribeServer/Controllers/AdminController.cs +++ b/ClassTranscribeServer/Controllers/AdminController.cs @@ -70,51 +70,55 @@ public ActionResult UpdateAllPlaylists() /// /// Regenerate one Caption (vtt, srt) file of the given Transcription /// - [HttpPost("UpdateVTTFile")] - [Authorize(Roles = Globals.ROLE_ADMIN)] - public ActionResult UpdateVTTFile(string transcriptionId) - { - _logger.LogInformation($"Enqueueing {transcriptionId} caption regeneration"); - _wakeDownloader.UpdateVTTFile(transcriptionId); - return Ok(); - } + /// will be deleted soon - We now generate vtt files dynamically. + // [HttpPost("UpdateVTTFile")] + // [Authorize(Roles = Globals.ROLE_ADMIN)] + // public ActionResult UpdateVTTFile(string transcriptionId) + // { + // _logger.LogInformation($"Enqueueing {transcriptionId} caption regeneration"); + // _wakeDownloader.UpdateVTTFile(transcriptionId); + // return Ok(); + // } /// /// Regenerate all Caption (vtt, srt) files of the given course offering /// - [HttpPost("UpdateVTTFilesInCourseOffering")] - [Authorize(Roles = Globals.ROLE_ADMIN)] - public async Task UpdateVTTFilesInCourseOffering(string offeringId = null) - { - - var playlistIds = await _context.Playlists.Where(p => p.OfferingId == offeringId).Select(p => p.Id).ToListAsync(); - _logger.LogInformation($"UpdateVTTFilesinPlaylist(${offeringId}): Found {playlistIds.Count} playlists"); - - var videoIds = await _context.Medias.Where(m => playlistIds.Contains(m.PlaylistId)).Select(m => m.VideoId).ToListAsync(); - _logger.LogInformation($"UpdateVTTFilesinPlaylist(): Found {videoIds.Count} videos"); - var transcriptionIds = await _context.Transcriptions.Where(t => videoIds.Contains(t.VideoId)).Select(t => t.Id).ToListAsync(); - _logger.LogInformation($"UpdateVTTFilesinPlaylist(): Found {transcriptionIds.Count} vtt transcriptions to regenerate"); - foreach (var t in transcriptionIds) - { - _wakeDownloader.UpdateVTTFile(t); - } - return Ok($"Requested {transcriptionIds.Count} Transcriptions to be regenerated from {videoIds.Count} videos in {playlistIds.Count} playlists"); - } + /// Will be deleted soon - we no longer store vtt files + // [HttpPost("UpdateVTTFilesInCourseOffering")] + // [Authorize(Roles = Globals.ROLE_ADMIN)] + // public async Task UpdateVTTFilesInCourseOffering(string offeringId = null) + // { + + // var playlistIds = await _context.Playlists.Where(p => p.OfferingId == offeringId).Select(p => p.Id).ToListAsync(); + // _logger.LogInformation($"UpdateVTTFilesinPlaylist(${offeringId}): Found {playlistIds.Count} playlists"); + + // var videoIds = await _context.Medias.Where(m => playlistIds.Contains(m.PlaylistId)).Select(m => m.VideoId).ToListAsync(); + // _logger.LogInformation($"UpdateVTTFilesinPlaylist(): Found {videoIds.Count} videos"); + // var transcriptionIds = await _context.Transcriptions.Where(t => videoIds.Contains(t.VideoId)).Select(t => t.Id).ToListAsync(); + // _logger.LogInformation($"UpdateVTTFilesinPlaylist(): Found {transcriptionIds.Count} vtt transcriptions to regenerate"); + // foreach (var t in transcriptionIds) + // { + // _wakeDownloader.UpdateVTTFile(t); + // } + // return Ok($"Requested {transcriptionIds.Count} Transcriptions to be regenerated from {videoIds.Count} videos in {playlistIds.Count} playlists"); + // } + /// /// Regenerate all Caption (vtt, srt) files of all transcriptions /// - [HttpPost("UpdateAllVTTFiles")] - [Authorize(Roles = Globals.ROLE_ADMIN)] - public async Task UpdateAllVTTFiles() - { - var transcriptionIds = await _context.Transcriptions.Select(t => t.Id).ToListAsync(); - _logger.LogInformation($"UpdateAllVTTFiles: Enqueueing {transcriptionIds.Count} vtt transcriptions to regenerate"); - foreach (var t in transcriptionIds) - { - _wakeDownloader.UpdateVTTFile(t); - } - return Ok(); - } + /// will be deleted soon - we no longer store vtt files + // [HttpPost("UpdateAllVTTFiles")] + // [Authorize(Roles = Globals.ROLE_ADMIN)] + // public async Task UpdateAllVTTFiles() + // { + // var transcriptionIds = await _context.Transcriptions.Select(t => t.Id).ToListAsync(); + // _logger.LogInformation($"UpdateAllVTTFiles: Enqueueing {transcriptionIds.Count} vtt transcriptions to regenerate"); + // foreach (var t in transcriptionIds) + // { + // _wakeDownloader.UpdateVTTFile(t); + // } + // return Ok(); + // } /// diff --git a/TaskEngine/Program.cs b/TaskEngine/Program.cs index 6492220..6588465 100644 --- a/TaskEngine/Program.cs +++ b/TaskEngine/Program.cs @@ -42,11 +42,14 @@ public static void Main() builder.AddConsole(); builder.AddFilter ("", LogLevel.Warning); - string insightKey = configuration.GetValue("APPLICATION_INSIGHTS_KEY"); - if (!String.IsNullOrEmpty(insightKey) && insightKey.Trim().Length>1) - { - builder.AddApplicationInsights(insightKey); - } + // If we use A.I. in the future - + // Use the AddApplicationInsights() overload which accepts Action and set TelemetryConfiguration.ConnectionString. See https://github.com/microsoft/ApplicationInsights-dotnet/issues/2560 for more details. + + // string insightKey = configuration.GetValue("APPLICATION_INSIGHTS_KEY"); + // if (!String.IsNullOrEmpty(insightKey) && insightKey.Trim().Length>1) + // { + // builder.AddApplicationInsights(insightKey); + // } }) .AddOptions() .Configure(configuration) @@ -58,7 +61,7 @@ public static void Main() .AddSingleton() .AddSingleton() .AddSingleton() - .AddSingleton() + // .AddSingleton() .AddSingleton() .AddSingleton() .AddSingleton() @@ -116,10 +119,11 @@ public static void Main() serviceProvider.GetService().Consume(concurrent_synctasks); // Transcription Related - _logger.LogInformation($"Creating TranscriptionTask & GenerateVTTFileTask consumers. Concurrency={concurrent_transcriptions} "); + _logger.LogInformation($"Creating TranscriptionTask consumers. Concurrency={concurrent_transcriptions} "); serviceProvider.GetService().Consume(concurrent_transcriptions); - serviceProvider.GetService().Consume(concurrent_transcriptions); + + // no more! - serviceProvider.GetService().Consume(concurrent_transcriptions); // Video Processing Related _logger.LogInformation($"Creating ProcessVideoTask consumer. Concurrency={concurrent_videotasks} "); diff --git a/TaskEngine/Tasks/DownloadMediaTask.cs b/TaskEngine/Tasks/DownloadMediaTask.cs index 9e6c37a..f43c052 100644 --- a/TaskEngine/Tasks/DownloadMediaTask.cs +++ b/TaskEngine/Tasks/DownloadMediaTask.cs @@ -134,17 +134,45 @@ protected async Task updateMediaWithVideo(string mediaId, Video newVideo) await newVideo.DeleteVideoAsync(_context); return false; } - GetLogger().LogInformation($"Media ({mediaId}): media.Video == null is {media.Video == null}"); + GetLogger().LogInformation($"Media ({mediaId}): existing media.Video {media.Video != null}"); + GetLogger().LogInformation($"Media ({mediaId}): media.Video?.Video1.Id={media.Video?.Video1.Id} ...Video2.Id={media.Video?.Video2.Id} "); + + GetLogger().LogInformation($"Media ({mediaId}): downloaded: newVideo.Video1={newVideo.Video1} ...Video2={newVideo.Video2} "); + GetLogger().LogInformation($"Media ({mediaId}): downloaded: newVideo.Video1.Hash={newVideo.Video1?.Hash} ...Hash2={newVideo.Video2?.Hash} "); - // Don't add video if there are already videos for the given media. + // if(newVideo.Id != null) { GetLogger().LogError($"Media ({mediaId}): Huh? newVideo should not have an Id yet - that's my job!"); + return false; } - if (media.Video != null) + + if (media.VideoId != null) { - GetLogger().LogInformation($"Media ({mediaId}): Surprise - media already has video set (race condition?)- no further processing required.Discarding new files"); + // Normally a DownloadMediaTask is only triggered if the video is null. + // So this code is run when a manual DownloadMediaTask is requested again + var changed = false; + var v = media.Video; + GetLogger().LogInformation($"Media ({mediaId}): Media already has video with video1Id <{media.VideoId}> Cherrypicking new files"); + var pickVideo2 = newVideo.Video2 != null && (v.Video2Id == null || newVideo.Video2.Hash != v.Video2.Hash); + GetLogger().LogInformation($"Media ({mediaId}):pickVideo2={pickVideo2}"); + + if( newVideo.Video2 != null && (v.Video2Id == null || newVideo.Video2.Hash != v.Video2.Hash)){ + _context.FileRecords.Add(newVideo.Video2); + _context.SaveChanges(); // now v2 has an Id, so we can use below + v.Video2 = newVideo.Video2; + newVideo.Video2 = null; + changed = true; + } + if(newVideo.ASLVideo != null && ( v.ASLVideoId == null || newVideo.ASLVideo.Hash != v.ASLVideo.Hash)) { + _context.FileRecords.Add(newVideo.ASLVideo); + _context.SaveChanges(); // now v2 has an Id, so we can use below + v.ASLVideo = newVideo.ASLVideo; + newVideo.ASLVideo = null; + changed = true; + } + if(changed) _context.SaveChanges(); await newVideo.DeleteVideoAsync(_context); - return false; + return changed; } // Time to find out what we have in the database // Important idea: the newVideo and its filerecords are not yet part of the database. @@ -155,7 +183,7 @@ protected async Task updateMediaWithVideo(string mediaId, Video newVideo) var existingPrimaryVideo = existingPrimaryVideos?.FirstOrDefault(); // If non null we expect 0 or 1 GetLogger().LogInformation($"Media ({mediaId}): {matchingFiles.Count} FileRecord hash match found"); - GetLogger().LogInformation($"Media ({mediaId}): {existingPrimaryVideos?.Count ?? 0} existing Videos found"); + GetLogger().LogInformation($"Media ({mediaId}): {existingPrimaryVideos?.Count ?? 0} existing Primary Videos found"); // cherrypick case (see comment below) if (existingPrimaryVideo != null) @@ -240,48 +268,67 @@ public async Task