From f1c642b4e9653701ce7dd7a92fc31053739729b0 Mon Sep 17 00:00:00 2001 From: Lawrence Angrave Date: Thu, 25 Jan 2024 21:11:11 -0600 Subject: [PATCH 1/3] Add Timestamp to NOMATCH --- .../Services/MSTranscription/MSTranscriptionService.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ClassTranscribeDatabase/Services/MSTranscription/MSTranscriptionService.cs b/ClassTranscribeDatabase/Services/MSTranscription/MSTranscriptionService.cs index 472aeb91..d42565b0 100644 --- a/ClassTranscribeDatabase/Services/MSTranscription/MSTranscriptionService.cs +++ b/ClassTranscribeDatabase/Services/MSTranscription/MSTranscriptionService.cs @@ -191,8 +191,8 @@ private async Task performRecognitionAsync(string logId, string fileP if (verboseLogging) { - _logger.LogInformation($"{logId}: Begin={begin.Minutes}:{begin.Seconds},{begin.Milliseconds}", begin); - _logger.LogInformation($"{logId}: End={end.Minutes}:{end.Seconds},{end.Milliseconds}"); + _logger.LogInformation($"{logId}: Begin={begin.Minutes}:{begin.Seconds}.{begin.Milliseconds}", begin); + _logger.LogInformation($"{logId}: End={end.Minutes}:{end.Seconds}.{end.Milliseconds}"); } // TODO/TOREVIEW: // ToCaptionEntitiesWithWordTiming vs ToCaptionEntitiesInterpolate @@ -243,7 +243,8 @@ private async Task performRecognitionAsync(string logId, string fileP } else if (e.Result.Reason == ResultReason.NoMatch) { - _logger.LogInformation($"{logId}: NOMATCH: Speech could not be recognized."); + TimeSpan begin = (new TimeSpan(e.Result.OffsetInTicks)).Add(restartOffset); + _logger.LogInformation($"{logId}: NOMATCH: ({begin.Minutes}:{begin.Seconds}) Speech could not be recognized."); } }; From 7423088fb9761c334311b1f7c0e8b1df9c2c4584 Mon Sep 17 00:00:00 2001 From: Lawrence Angrave Date: Thu, 25 Jan 2024 21:12:55 -0600 Subject: [PATCH 2/3] Remove VTT Task --- .../Controllers/AdminController.cs | 80 +++++++++-------- TaskEngine/Program.cs | 20 +++-- TaskEngine/Tasks/GenerateVTTFileTask.cs | 86 +++++++++---------- TaskEngine/Tasks/QueueAwakerTask.cs | 54 ++++++------ TaskEngine/Tasks/TranscriptionTask.cs | 9 +- TaskEngine/TempCode.cs | 6 +- 6 files changed, 133 insertions(+), 122 deletions(-) diff --git a/ClassTranscribeServer/Controllers/AdminController.cs b/ClassTranscribeServer/Controllers/AdminController.cs index 1ad459a6..f9a88a70 100644 --- a/ClassTranscribeServer/Controllers/AdminController.cs +++ b/ClassTranscribeServer/Controllers/AdminController.cs @@ -70,51 +70,55 @@ public ActionResult UpdateAllPlaylists() /// /// Regenerate one Caption (vtt, srt) file of the given Transcription /// - [HttpPost("UpdateVTTFile")] - [Authorize(Roles = Globals.ROLE_ADMIN)] - public ActionResult UpdateVTTFile(string transcriptionId) - { - _logger.LogInformation($"Enqueueing {transcriptionId} caption regeneration"); - _wakeDownloader.UpdateVTTFile(transcriptionId); - return Ok(); - } + /// will be deleted soon - We now generate vtt files dynamically. + // [HttpPost("UpdateVTTFile")] + // [Authorize(Roles = Globals.ROLE_ADMIN)] + // public ActionResult UpdateVTTFile(string transcriptionId) + // { + // _logger.LogInformation($"Enqueueing {transcriptionId} caption regeneration"); + // _wakeDownloader.UpdateVTTFile(transcriptionId); + // return Ok(); + // } /// /// Regenerate all Caption (vtt, srt) files of the given course offering /// - [HttpPost("UpdateVTTFilesInCourseOffering")] - [Authorize(Roles = Globals.ROLE_ADMIN)] - public async Task UpdateVTTFilesInCourseOffering(string offeringId = null) - { - - var playlistIds = await _context.Playlists.Where(p => p.OfferingId == offeringId).Select(p => p.Id).ToListAsync(); - _logger.LogInformation($"UpdateVTTFilesinPlaylist(${offeringId}): Found {playlistIds.Count} playlists"); - - var videoIds = await _context.Medias.Where(m => playlistIds.Contains(m.PlaylistId)).Select(m => m.VideoId).ToListAsync(); - _logger.LogInformation($"UpdateVTTFilesinPlaylist(): Found {videoIds.Count} videos"); - var transcriptionIds = await _context.Transcriptions.Where(t => videoIds.Contains(t.VideoId)).Select(t => t.Id).ToListAsync(); - _logger.LogInformation($"UpdateVTTFilesinPlaylist(): Found {transcriptionIds.Count} vtt transcriptions to regenerate"); - foreach (var t in transcriptionIds) - { - _wakeDownloader.UpdateVTTFile(t); - } - return Ok($"Requested {transcriptionIds.Count} Transcriptions to be regenerated from {videoIds.Count} videos in {playlistIds.Count} playlists"); - } + /// Will be deleted soon - we no longer store vtt files + // [HttpPost("UpdateVTTFilesInCourseOffering")] + // [Authorize(Roles = Globals.ROLE_ADMIN)] + // public async Task UpdateVTTFilesInCourseOffering(string offeringId = null) + // { + + // var playlistIds = await _context.Playlists.Where(p => p.OfferingId == offeringId).Select(p => p.Id).ToListAsync(); + // _logger.LogInformation($"UpdateVTTFilesinPlaylist(${offeringId}): Found {playlistIds.Count} playlists"); + + // var videoIds = await _context.Medias.Where(m => playlistIds.Contains(m.PlaylistId)).Select(m => m.VideoId).ToListAsync(); + // _logger.LogInformation($"UpdateVTTFilesinPlaylist(): Found {videoIds.Count} videos"); + // var transcriptionIds = await _context.Transcriptions.Where(t => videoIds.Contains(t.VideoId)).Select(t => t.Id).ToListAsync(); + // _logger.LogInformation($"UpdateVTTFilesinPlaylist(): Found {transcriptionIds.Count} vtt transcriptions to regenerate"); + // foreach (var t in transcriptionIds) + // { + // _wakeDownloader.UpdateVTTFile(t); + // } + // return Ok($"Requested {transcriptionIds.Count} Transcriptions to be regenerated from {videoIds.Count} videos in {playlistIds.Count} playlists"); + // } + /// /// Regenerate all Caption (vtt, srt) files of all transcriptions /// - [HttpPost("UpdateAllVTTFiles")] - [Authorize(Roles = Globals.ROLE_ADMIN)] - public async Task UpdateAllVTTFiles() - { - var transcriptionIds = await _context.Transcriptions.Select(t => t.Id).ToListAsync(); - _logger.LogInformation($"UpdateAllVTTFiles: Enqueueing {transcriptionIds.Count} vtt transcriptions to regenerate"); - foreach (var t in transcriptionIds) - { - _wakeDownloader.UpdateVTTFile(t); - } - return Ok(); - } + /// will be deleted soon - we no longer store vtt files + // [HttpPost("UpdateAllVTTFiles")] + // [Authorize(Roles = Globals.ROLE_ADMIN)] + // public async Task UpdateAllVTTFiles() + // { + // var transcriptionIds = await _context.Transcriptions.Select(t => t.Id).ToListAsync(); + // _logger.LogInformation($"UpdateAllVTTFiles: Enqueueing {transcriptionIds.Count} vtt transcriptions to regenerate"); + // foreach (var t in transcriptionIds) + // { + // _wakeDownloader.UpdateVTTFile(t); + // } + // return Ok(); + // } /// diff --git a/TaskEngine/Program.cs b/TaskEngine/Program.cs index 6492220a..6588465f 100644 --- a/TaskEngine/Program.cs +++ b/TaskEngine/Program.cs @@ -42,11 +42,14 @@ public static void Main() builder.AddConsole(); builder.AddFilter ("", LogLevel.Warning); - string insightKey = configuration.GetValue("APPLICATION_INSIGHTS_KEY"); - if (!String.IsNullOrEmpty(insightKey) && insightKey.Trim().Length>1) - { - builder.AddApplicationInsights(insightKey); - } + // If we use A.I. in the future - + // Use the AddApplicationInsights() overload which accepts Action and set TelemetryConfiguration.ConnectionString. See https://github.com/microsoft/ApplicationInsights-dotnet/issues/2560 for more details. + + // string insightKey = configuration.GetValue("APPLICATION_INSIGHTS_KEY"); + // if (!String.IsNullOrEmpty(insightKey) && insightKey.Trim().Length>1) + // { + // builder.AddApplicationInsights(insightKey); + // } }) .AddOptions() .Configure(configuration) @@ -58,7 +61,7 @@ public static void Main() .AddSingleton() .AddSingleton() .AddSingleton() - .AddSingleton() + // .AddSingleton() .AddSingleton() .AddSingleton() .AddSingleton() @@ -116,10 +119,11 @@ public static void Main() serviceProvider.GetService().Consume(concurrent_synctasks); // Transcription Related - _logger.LogInformation($"Creating TranscriptionTask & GenerateVTTFileTask consumers. Concurrency={concurrent_transcriptions} "); + _logger.LogInformation($"Creating TranscriptionTask consumers. Concurrency={concurrent_transcriptions} "); serviceProvider.GetService().Consume(concurrent_transcriptions); - serviceProvider.GetService().Consume(concurrent_transcriptions); + + // no more! - serviceProvider.GetService().Consume(concurrent_transcriptions); // Video Processing Related _logger.LogInformation($"Creating ProcessVideoTask consumer. Concurrency={concurrent_videotasks} "); diff --git a/TaskEngine/Tasks/GenerateVTTFileTask.cs b/TaskEngine/Tasks/GenerateVTTFileTask.cs index 84d74184..7c7e00ee 100644 --- a/TaskEngine/Tasks/GenerateVTTFileTask.cs +++ b/TaskEngine/Tasks/GenerateVTTFileTask.cs @@ -31,57 +31,57 @@ protected async override Task OnConsume(string transcriptionId, TaskParameters t return; - using (var _context = CTDbContext.CreateDbContext()) - { - var transcription = await _context.Transcriptions.FindAsync(transcriptionId); - string subdir = ToCourseOfferingSubDirectory(_context, transcription); - CaptionQueries captionQueries = new CaptionQueries(_context); - var captions = await captionQueries.GetCaptionsAsync(transcription.Id); +// using (var _context = CTDbContext.CreateDbContext()) +// { +// var transcription = await _context.Transcriptions.FindAsync(transcriptionId); +// string subdir = ToCourseOfferingSubDirectory(_context, transcription); +// CaptionQueries captionQueries = new CaptionQueries(_context); +// var captions = await captionQueries.GetCaptionsAsync(transcription.Id); - var vttfile = await FileRecord.GetNewFileRecordAsync(Caption.GenerateWebVTTFile(captions, transcription.Language), ".vtt", subdir); +// var vttfile = await FileRecord.GetNewFileRecordAsync(Caption.GenerateWebVTTFile(captions, transcription.Language), ".vtt", subdir); -#nullable enable - FileRecord? existingVtt = await _context.FileRecords.FindAsync(transcription.FileId); -#nullable disable +// #nullable enable +// FileRecord? existingVtt = await _context.FileRecords.FindAsync(transcription.FileId); +// #nullable disable - if (existingVtt is null) - { - GetLogger().LogInformation($"{transcriptionId}: Creating new vtt file {vttfile.FileName}"); - await _context.FileRecords.AddAsync(vttfile); - transcription.File = vttfile; - _context.Entry(transcription).State = EntityState.Modified; - } - else - { - GetLogger().LogInformation($"{transcriptionId}: replacing existing vtt file contents {existingVtt.FileName}"); - existingVtt.ReplaceWith(vttfile); - _context.Entry(existingVtt).State = EntityState.Modified; - } +// if (existingVtt is null) +// { +// GetLogger().LogInformation($"{transcriptionId}: Creating new vtt file {vttfile.FileName}"); +// await _context.FileRecords.AddAsync(vttfile); +// transcription.File = vttfile; +// _context.Entry(transcription).State = EntityState.Modified; +// } +// else +// { +// GetLogger().LogInformation($"{transcriptionId}: replacing existing vtt file contents {existingVtt.FileName}"); +// existingVtt.ReplaceWith(vttfile); +// _context.Entry(existingVtt).State = EntityState.Modified; +// } - var srtfile = await FileRecord.GetNewFileRecordAsync(Caption.GenerateSrtFile(captions), ".srt", subdir); +// var srtfile = await FileRecord.GetNewFileRecordAsync(Caption.GenerateSrtFile(captions), ".srt", subdir); -#nullable enable - FileRecord? existingSrt = await _context.FileRecords.FindAsync(transcription.SrtFileId); -#nullable disable +// #nullable enable +// FileRecord? existingSrt = await _context.FileRecords.FindAsync(transcription.SrtFileId); +// #nullable disable - if (existingSrt is null) - { - GetLogger().LogInformation($"{transcriptionId}: Creating new srt file {srtfile.FileName}"); +// if (existingSrt is null) +// { +// GetLogger().LogInformation($"{transcriptionId}: Creating new srt file {srtfile.FileName}"); - await _context.FileRecords.AddAsync(srtfile); - transcription.SrtFile = srtfile; - _context.Entry(transcription).State = EntityState.Modified; - } - else - { - GetLogger().LogInformation($"{transcriptionId}: replacing existing srt file contents {existingSrt.FileName}"); - existingSrt.ReplaceWith(srtfile); - _context.Entry(existingSrt).State = EntityState.Modified; - } +// await _context.FileRecords.AddAsync(srtfile); +// transcription.SrtFile = srtfile; +// _context.Entry(transcription).State = EntityState.Modified; +// } +// else +// { +// GetLogger().LogInformation($"{transcriptionId}: replacing existing srt file contents {existingSrt.FileName}"); +// existingSrt.ReplaceWith(srtfile); +// _context.Entry(existingSrt).State = EntityState.Modified; +// } - await _context.SaveChangesAsync(); - GetLogger().LogInformation($"{transcriptionId}: Database updated"); - } +// await _context.SaveChangesAsync(); +// GetLogger().LogInformation($"{transcriptionId}: Database updated"); +// } } } } \ No newline at end of file diff --git a/TaskEngine/Tasks/QueueAwakerTask.cs b/TaskEngine/Tasks/QueueAwakerTask.cs index caff1381..a6167f99 100644 --- a/TaskEngine/Tasks/QueueAwakerTask.cs +++ b/TaskEngine/Tasks/QueueAwakerTask.cs @@ -23,7 +23,7 @@ class QueueAwakerTask : RabbitMQTask private readonly DownloadMediaTask _downloadMediaTask; // private readonly ConvertVideoToWavTask _convertVideoToWavTask; private readonly TranscriptionTask _transcriptionTask; - private readonly GenerateVTTFileTask _generateVTTFileTask; + // nope private readonly GenerateVTTFileTask _generateVTTFileTask; private readonly ProcessVideoTask _processVideoTask; private readonly SceneDetectionTask _sceneDetectionTask; private readonly PythonCrawlerTask _pythonCrawlerTask; @@ -40,7 +40,8 @@ public QueueAwakerTask() { } public QueueAwakerTask(RabbitMQConnection rabbitMQ, DownloadPlaylistInfoTask downloadPlaylistInfoTask, DownloadMediaTask downloadMediaTask, TranscriptionTask transcriptionTask, ProcessVideoTask processVideoTask, - GenerateVTTFileTask generateVTTFileTask, SceneDetectionTask sceneDetectionTask, + // GenerateVTTFileTask generateVTTFileTask, + SceneDetectionTask sceneDetectionTask, CreateBoxTokenTask createBoxTokenTask, UpdateBoxTokenTask updateBoxTokenTask, PythonCrawlerTask pythonCrawlerTask, BuildElasticIndexTask buildElasticIndexTask, CleanUpElasticIndexTask cleanUpElasticIndexTask, @@ -52,7 +53,7 @@ public QueueAwakerTask(RabbitMQConnection rabbitMQ, DownloadPlaylistInfoTask dow _downloadMediaTask = downloadMediaTask; //_convertVideoToWavTask = convertVideoToWavTask; _transcriptionTask = transcriptionTask; - _generateVTTFileTask = generateVTTFileTask; + // _generateVTTFileTask = generateVTTFileTask; _processVideoTask = processVideoTask; _sceneDetectionTask = sceneDetectionTask; _pythonCrawlerTask = pythonCrawlerTask; @@ -147,7 +148,7 @@ private async Task PendingJobs() _updateBoxTokenTask.Publish(""); //We will use these outside of the DB scope - List todoVTTs ; + // List todoVTTs ; List todoProcessVideos; List todoTranscriptions; List todoDownloads; @@ -183,7 +184,7 @@ private async Task PendingJobs() // Completed Transcriptions which haven't generated vtt files // TODO: Should also check dates too - GetLogger().LogInformation($"Finding incomplete VTTs, Transcriptions and Downloads from before {tooRecentCutoff}, minutesCutOff=({minutesCutOff})"); + GetLogger().LogInformation($"Finding incomplete Transcriptions and Downloads from before {tooRecentCutoff}, minutesCutOff=({minutesCutOff})"); // Todo Could also check for secondary video too @@ -192,10 +193,11 @@ private async Task PendingJobs() v=>(v.Duration == null && ! String.IsNullOrEmpty(v.Video1Id)) ).OrderByDescending(t => t.CreatedAt).Take(maxProcessVideos).Select(e => e.Id).ToListAsync(); - var maxVTTs = 400; - todoVTTs = await context.Transcriptions.AsNoTracking().Where( - t => t.Captions.Count > 0 && t.File == null && t.CreatedAt < tooRecentCutoff - ).OrderByDescending(t => t.CreatedAt).Take(maxVTTs).Select(e => e.Id).ToListAsync(); + // We no longer create vtt files + // var maxVTTs = 400; + // todoVTTs = await context.Transcriptions.AsNoTracking().Where( + // t => t.Captions.Count > 0 && t.File == null && t.CreatedAt < tooRecentCutoff + // ).OrderByDescending(t => t.CreatedAt).Take(maxVTTs).Select(e => e.Id).ToListAsync(); var maxSceneDetection = 400; todoSceneDetection = await context.Videos.AsNoTracking().Where( @@ -219,14 +221,13 @@ private async Task PendingJobs() // However some of these may already be in progress // So don't queue theses - GetLogger().LogInformation($"Found {todoProcessVideos.Count},{todoVTTs.Count},{todoTranscriptions.Count},{todoDownloads.Count} counts before filtering"); + GetLogger().LogInformation($"Found todoProcessVideos={todoProcessVideos.Count},todoTranscriptions={todoTranscriptions.Count},todoDownloads={todoDownloads.Count} counts before filtering"); ClientActiveTasks currentProcessVideos = _processVideoTask.GetCurrentTasks(); todoProcessVideos.RemoveAll(e => currentProcessVideos.Contains(e)); - ClientActiveTasks currentVTTs = _generateVTTFileTask.GetCurrentTasks(); - todoVTTs.RemoveAll(e => currentVTTs.Contains(e)); - + // ClientActiveTasks currentVTTs = _generateVTTFileTask.GetCurrentTasks(); + // todoVTTs.RemoveAll(e => currentVTTs.Contains(e)); ClientActiveTasks currentSceneDetection = _sceneDetectionTask.GetCurrentTasks(); todoSceneDetection.RemoveAll(e => currentSceneDetection.Contains(e)); @@ -237,8 +238,8 @@ private async Task PendingJobs() ClientActiveTasks currentDownloads = _transcriptionTask.GetCurrentTasks(); todoDownloads.RemoveAll(e => currentDownloads.Contains(e)); - GetLogger().LogInformation($"Current In progress {currentProcessVideos.Count},{currentVTTs.Count},{currentTranscription.Count},{currentDownloads.Count} counts after filtering"); - GetLogger().LogInformation($"Found {todoProcessVideos.Count},{todoVTTs.Count},{todoTranscriptions.Count},{todoDownloads.Count} counts after filtering"); + GetLogger().LogInformation($"Current In progress todoProcessVideos={currentProcessVideos.Count},currentTranscription={currentTranscription.Count},currentDownloads={currentDownloads.Count} counts after filtering"); + GetLogger().LogInformation($"Found todoProcessVideos={todoProcessVideos.Count},todoTranscriptions={todoTranscriptions.Count},todoDownloads={todoDownloads.Count} counts after filtering"); // Now we have a list of new things we want to do @@ -251,9 +252,8 @@ private async Task PendingJobs() GetLogger().LogInformation($"Publishing SceneDetects ({String.Join(",", todoSceneDetection)})"); todoSceneDetection.ForEach(t => _sceneDetectionTask.Publish(t)); - GetLogger().LogInformation($"Publishing todoVTTs ({String.Join(",", todoVTTs)})"); - - todoVTTs.ForEach(t => _generateVTTFileTask.Publish(t)); + // GetLogger().LogInformation($"Publishing todoVTTs ({String.Join(",", todoVTTs)})"); + // todoVTTs.ForEach(t => _generateVTTFileTask.Publish(t)); GetLogger().LogInformation($"Publishing todoTranscriptions ({String.Join(",", todoTranscriptions)})"); @@ -316,14 +316,16 @@ protected async override Task OnConsume(JObject jObject, TaskParameters taskPara var playlist = await _context.Playlists.FindAsync(playlistId); _downloadPlaylistInfoTask.Publish(playlist.Id); } - else if (type == TaskType.GenerateVTTFile.ToString()) - { - var transcriptionId = jObject["TranscriptionId"].ToString(); - var transcription = await _context.Transcriptions.FindAsync(transcriptionId); - _generateVTTFileTask.Publish(transcription.Id); - } - - else if (type == TaskType.CreateBoxToken.ToString()) + else + // if (type == TaskType.GenerateVTTFile.ToString()) + // { + // var transcriptionId = jObject["TranscriptionId"].ToString(); + // var transcription = await _context.Transcriptions.FindAsync(transcriptionId); + // _generateVTTFileTask.Publish(transcription.Id); + // } + + // else + if (type == TaskType.CreateBoxToken.ToString()) { var authCode = jObject["authCode"].ToString(); _createBoxTokenTask.Publish(authCode); diff --git a/TaskEngine/Tasks/TranscriptionTask.cs b/TaskEngine/Tasks/TranscriptionTask.cs index 22de3871..217fa633 100644 --- a/TaskEngine/Tasks/TranscriptionTask.cs +++ b/TaskEngine/Tasks/TranscriptionTask.cs @@ -25,16 +25,17 @@ class TranscriptionTask : RabbitMQTask { private readonly MSTranscriptionService _msTranscriptionService; - private readonly GenerateVTTFileTask _generateVTTFileTask; + // nope private readonly GenerateVTTFileTask _generateVTTFileTask; private readonly CaptionQueries _captionQueries; public TranscriptionTask(RabbitMQConnection rabbitMQ, MSTranscriptionService msTranscriptionService, - GenerateVTTFileTask generateVTTFileTask, ILogger logger, CaptionQueries captionQueries) + // GenerateVTTFileTask generateVTTFileTask, + ILogger logger, CaptionQueries captionQueries) : base(rabbitMQ, TaskType.TranscribeVideo, logger) { _msTranscriptionService = msTranscriptionService; - _generateVTTFileTask = generateVTTFileTask; + // nope _generateVTTFileTask = generateVTTFileTask; _captionQueries = captionQueries; } @@ -249,7 +250,7 @@ protected async override Task OnConsume(string videoId, TaskParameters taskParam GetLogger().LogInformation($"{videoId}: Saving captions Code={result.ErrorCode}. LastSuccessTime={result.LastSuccessTime}"); await _context.SaveChangesAsync(); - video.Transcriptions.ForEach(t => _generateVTTFileTask.Publish(t.Id)); + // nope, no vtt files video.Transcriptions.ForEach(t => _generateVTTFileTask.Publish(t.Id)); } catch (Exception ex) diff --git a/TaskEngine/TempCode.cs b/TaskEngine/TempCode.cs index 9a9989f7..d2c50f2a 100644 --- a/TaskEngine/TempCode.cs +++ b/TaskEngine/TempCode.cs @@ -23,7 +23,7 @@ class TempCode private readonly SceneDetectionTask _sceneDetectionTask; private readonly PythonCrawlerTask _pythonCrawlerTask; private readonly ProcessVideoTask _processVideoTask; - private readonly GenerateVTTFileTask _generateVTTFileTask; + // private readonly GenerateVTTFileTask _generateVTTFileTask; private readonly TranscriptionTask _transcriptionTask; private readonly ConvertVideoToWavTask _convertVideoToWavTask; private readonly DownloadMediaTask _downloadMediaTask; @@ -33,7 +33,7 @@ class TempCode private readonly CleanUpElasticIndexTask _cleanUpElasticIndexTask; public TempCode(CTDbContext c, CreateBoxTokenTask createBoxTokenTask, UpdateBoxTokenTask updateBoxTokenTask, - SceneDetectionTask ePubGeneratorTask, ProcessVideoTask processVideoTask, GenerateVTTFileTask generateVTTFileTask, + SceneDetectionTask ePubGeneratorTask, ProcessVideoTask processVideoTask, TranscriptionTask transcriptionTask, ConvertVideoToWavTask convertVideoToWavTask, DownloadMediaTask downloadMediaTask, DownloadPlaylistInfoTask downloadPlaylistInfoTask, QueueAwakerTask queueAwakerTask, CleanUpElasticIndexTask cleanUpElasticIndexTask, RpcClient rpcClient, @@ -44,7 +44,7 @@ public TempCode(CTDbContext c, CreateBoxTokenTask createBoxTokenTask, UpdateBoxT _updateBoxTokenTask = updateBoxTokenTask; _sceneDetectionTask = ePubGeneratorTask; _processVideoTask = processVideoTask; - _generateVTTFileTask = generateVTTFileTask; + // _generateVTTFileTask = generateVTTFileTask; _transcriptionTask = transcriptionTask; _convertVideoToWavTask = convertVideoToWavTask; _downloadMediaTask = downloadMediaTask; From 45db2efcd786b493a1deb0482fdec81fe46f6d5d Mon Sep 17 00:00:00 2001 From: Lawrence Angrave Date: Thu, 25 Jan 2024 21:16:56 -0600 Subject: [PATCH 3/3] Support DownloadMedia when video exists --- TaskEngine/Tasks/DownloadMediaTask.cs | 83 +++++++++++++++++++++------ 1 file changed, 65 insertions(+), 18 deletions(-) diff --git a/TaskEngine/Tasks/DownloadMediaTask.cs b/TaskEngine/Tasks/DownloadMediaTask.cs index 9e6c37a7..f43c0528 100644 --- a/TaskEngine/Tasks/DownloadMediaTask.cs +++ b/TaskEngine/Tasks/DownloadMediaTask.cs @@ -134,17 +134,45 @@ protected async Task updateMediaWithVideo(string mediaId, Video newVideo) await newVideo.DeleteVideoAsync(_context); return false; } - GetLogger().LogInformation($"Media ({mediaId}): media.Video == null is {media.Video == null}"); + GetLogger().LogInformation($"Media ({mediaId}): existing media.Video {media.Video != null}"); + GetLogger().LogInformation($"Media ({mediaId}): media.Video?.Video1.Id={media.Video?.Video1.Id} ...Video2.Id={media.Video?.Video2.Id} "); + + GetLogger().LogInformation($"Media ({mediaId}): downloaded: newVideo.Video1={newVideo.Video1} ...Video2={newVideo.Video2} "); + GetLogger().LogInformation($"Media ({mediaId}): downloaded: newVideo.Video1.Hash={newVideo.Video1?.Hash} ...Hash2={newVideo.Video2?.Hash} "); - // Don't add video if there are already videos for the given media. + // if(newVideo.Id != null) { GetLogger().LogError($"Media ({mediaId}): Huh? newVideo should not have an Id yet - that's my job!"); + return false; } - if (media.Video != null) + + if (media.VideoId != null) { - GetLogger().LogInformation($"Media ({mediaId}): Surprise - media already has video set (race condition?)- no further processing required.Discarding new files"); + // Normally a DownloadMediaTask is only triggered if the video is null. + // So this code is run when a manual DownloadMediaTask is requested again + var changed = false; + var v = media.Video; + GetLogger().LogInformation($"Media ({mediaId}): Media already has video with video1Id <{media.VideoId}> Cherrypicking new files"); + var pickVideo2 = newVideo.Video2 != null && (v.Video2Id == null || newVideo.Video2.Hash != v.Video2.Hash); + GetLogger().LogInformation($"Media ({mediaId}):pickVideo2={pickVideo2}"); + + if( newVideo.Video2 != null && (v.Video2Id == null || newVideo.Video2.Hash != v.Video2.Hash)){ + _context.FileRecords.Add(newVideo.Video2); + _context.SaveChanges(); // now v2 has an Id, so we can use below + v.Video2 = newVideo.Video2; + newVideo.Video2 = null; + changed = true; + } + if(newVideo.ASLVideo != null && ( v.ASLVideoId == null || newVideo.ASLVideo.Hash != v.ASLVideo.Hash)) { + _context.FileRecords.Add(newVideo.ASLVideo); + _context.SaveChanges(); // now v2 has an Id, so we can use below + v.ASLVideo = newVideo.ASLVideo; + newVideo.ASLVideo = null; + changed = true; + } + if(changed) _context.SaveChanges(); await newVideo.DeleteVideoAsync(_context); - return false; + return changed; } // Time to find out what we have in the database // Important idea: the newVideo and its filerecords are not yet part of the database. @@ -155,7 +183,7 @@ protected async Task updateMediaWithVideo(string mediaId, Video newVideo) var existingPrimaryVideo = existingPrimaryVideos?.FirstOrDefault(); // If non null we expect 0 or 1 GetLogger().LogInformation($"Media ({mediaId}): {matchingFiles.Count} FileRecord hash match found"); - GetLogger().LogInformation($"Media ({mediaId}): {existingPrimaryVideos?.Count ?? 0} existing Videos found"); + GetLogger().LogInformation($"Media ({mediaId}): {existingPrimaryVideos?.Count ?? 0} existing Primary Videos found"); // cherrypick case (see comment below) if (existingPrimaryVideo != null) @@ -240,48 +268,67 @@ public async Task