From b74a6cfa4fc0245117a62003f96274f26062f545 Mon Sep 17 00:00:00 2001 From: Lawrence Angrave Date: Wed, 17 Jan 2024 21:58:10 -0600 Subject: [PATCH 1/9] Update PythonRPCServer Requirements.txt --- PythonRpcServer/requirements.txt | 81 ++++++++++++++++++-------------- 1 file changed, 45 insertions(+), 36 deletions(-) diff --git a/PythonRpcServer/requirements.txt b/PythonRpcServer/requirements.txt index 0de6e7c..533936f 100644 --- a/PythonRpcServer/requirements.txt +++ b/PythonRpcServer/requirements.txt @@ -1,42 +1,52 @@ -backcall==0.1.0 -certifi==2023.7.22 -chardet==3.0.4 -click==7.1.1 -decorator==4.4.2 -ffmpy==0.2.2 -grpcio==1.28.1 -grpcio-tools==1.28.1 -idna==2.9 -ipython==7.16.3 -ipython-genutils==0.2.0 -jedi==0.17.0 -KalturaApiClient==18.11.0 -lxml==4.9.1 - -#numpy==1.21.6 -numpy -#Dependabot suggested 1.22 ; but that's not generally available yet +# 2024-1-17 Removed Pygments,python-genutils and ipython +# Also removed jedi. why autocomplete?? jedi==0.19.1 +#Why? prompt-toolkit==3.0.43 + +#Did not try updating (maybe nexttime) +#protobuf==3.15.0 #4.25.2 +protobuf==4.25.2 -#opencv-python==4.2.0.34 -# replaced by opencv-contrib-python -parso==0.7.0 -pexpect==4.8.0 +#Floowing Updated to latest 2024-1-17: +certifi==2023.11.17 +backcall==0.2.0 +chardet==5.2.0 +click==8.1.7 +decorator==5.1.1 +ffmpy==0.3.1 +grpcio==1.60.0 +grpcio-tools==1.60.0 +idna==3.6 +KalturaApiClient==19.3.0 +lxml==5.1.0 +parso==0.8.3 +pexpect==4.9.0 pickleshare==0.7.5 -prompt-toolkit==3.0.5 -protobuf==3.15.0 +ptyprocess==0.7.0 +requests==2.31.0 +requests-toolbelt==1.0.0 +six==1.16.0 +tqdm==4.66.1 +traitlets==4.3.3 +urllib3==2.1.0 +wcwidth==0.2.13 + +# Not versioned +numpy +pytube # if not available, use the tar.gz package (see Dockerfile) + + # protobuf version 3.18.3 causes NotImplementedError("To be implemented") in PythonRpcServer/mediaprovider.py # Likely need to coordinate updating the C# version too -ptyprocess==0.6.0 -Pygments==2.7.4 -requests==2.23.0 -requests-toolbelt==0.9.1 + +############# +# Gone +#ipython==7.16.3 +#ipython-genutils==0.2.0 +# Pygments==2.7.4 + +# No longer needed for remaning pythonrpcserver tasks #scenedetect==0.5.2 -six==1.14.0 -tqdm==4.45.0 -traitlets==4.3.3 -urllib3==1.25.8 -wcwidth==0.1.9 #scikit-image==0.17.2 #nltk==3.6.6 #pytesseract==0.3.7 @@ -44,6 +54,5 @@ wcwidth==0.1.9 #opencv-contrib-python==4.5.3.56 #mtcnn-opencv==1.0.2 #decord==0.6.0 - -# Use latest version -pytube # if not available, use the tar.gz package (see Dockerfile) +#opencv-python==4.2.0.34; replaced by opencv-contrib-python +# From e4a3e49d72d0a4ad9a98fb86a8915ccc5a5262b1 Mon Sep 17 00:00:00 2001 From: Lawrence Angrave Date: Wed, 17 Jan 2024 22:58:20 -0600 Subject: [PATCH 2/9] Update API and Taskengine Dockerfiles --- API.Dockerfile | 26 +++++++++++++++++++++++--- TaskEngine.Dockerfile | 12 ++++++------ 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/API.Dockerfile b/API.Dockerfile index eebe181..a9bffea 100644 --- a/API.Dockerfile +++ b/API.Dockerfile @@ -1,14 +1,34 @@ -FROM mcr.microsoft.com/dotnet/sdk:8.0.100-1-bookworm-slim as build +# Also remove platform from docker-compose.override.yml for api and taskengine +# Uncomment build context in docker-compose.override.yml for api and taskengine + +# e.g., +# taskengine: +# image: classtranscribe/taskengine:staging +# #xx platform: linux/amd64 # Nope - Causes SDK "dotnet restore" to hang on M1 Mac +# build: +# context: ../../WebAPI +# target: publish +# dockerfile: ./TaskEngine.Dockerfile +# + + +#FROM mcr.microsoft.com/dotnet/sdk:8.0-bookworm-slim-amd64 as build +FROM mcr.microsoft.com/dotnet/sdk:8.0-bookworm-slim as build # See https://mcr.microsoft.com/en-us/product/dotnet/sdk/tags -# FROM mcr.microsoft.com/dotnet/core/sdk:3.1.201-bionic as build +# Running the AMD64 version is of the SDK is broken +# https://github.com/dotnet/dotnet-docker/discussions/4285 +# https://github.com/NuGet/Home/issues/13062 + RUN apt-get -q update && apt-get -qy install git WORKDIR / RUN git clone https://github.com/eficode/wait-for.git WORKDIR /src COPY ./ClassTranscribeDatabase/ClassTranscribeDatabase.csproj ./ClassTranscribeDatabase/ClassTranscribeDatabase.csproj -RUN dotnet restore ./ClassTranscribeDatabase/ClassTranscribeDatabase.csproj +# Did not help ENV DOTNET_NUGET_SIGNATURE_VERIFICATION=false +# Add --verbosity normal|diagnostic +RUN dotnet restore --verbosity diagnostic ./ClassTranscribeDatabase/ClassTranscribeDatabase.csproj COPY ./ClassTranscribeServer/ClassTranscribeServer.csproj ./ClassTranscribeServer/ClassTranscribeServer.csproj RUN dotnet restore ./ClassTranscribeServer/ClassTranscribeServer.csproj diff --git a/TaskEngine.Dockerfile b/TaskEngine.Dockerfile index bf6c984..ce64dea 100644 --- a/TaskEngine.Dockerfile +++ b/TaskEngine.Dockerfile @@ -1,14 +1,14 @@ -FROM mcr.microsoft.com/dotnet/sdk:8.0.100-1-bookworm-slim as build +FROM mcr.microsoft.com/dotnet/sdk:8.0-bookworm-slim as build # See https://mcr.microsoft.com/en-us/product/dotnet/sdk/tags -# 7.0.404-1 as build -# FROM mcr.microsoft.com/dotnet/core/sdk:3.1.201-bionic as build +#See more comments in API.Dockerfile WORKDIR / RUN git clone https://github.com/eficode/wait-for.git WORKDIR /src COPY ./ClassTranscribeDatabase/ClassTranscribeDatabase.csproj ./ClassTranscribeDatabase/ClassTranscribeDatabase.csproj -RUN dotnet restore ./ClassTranscribeDatabase/ClassTranscribeDatabase.csproj +# --verbosity normal|diagnostic +RUN dotnet restore --verbosity diagnostic ./ClassTranscribeDatabase/ClassTranscribeDatabase.csproj COPY ./TaskEngine/TaskEngine.csproj ./TaskEngine/TaskEngine.csproj RUN dotnet restore ./TaskEngine/TaskEngine.csproj @@ -20,8 +20,8 @@ COPY ./TaskEngine ./TaskEngine WORKDIR /src/TaskEngine RUN dotnet publish TaskEngine.csproj -c Release -o /app --no-restore -FROM mcr.microsoft.com/dotnet/aspnet:8.0-bookworm-slim as publish_base -# FROM mcr.microsoft.com/dotnet/core/runtime:3.1.3-bionic as publish_base +FROM mcr.microsoft.com/dotnet/aspnet:8.0-bookworm-slim-amd64 as publish_base +# force AMD64 build here: the ssl1.1.1 workaround below assumes amd64 # Install prerequisites for Azure Speech Services: build-essential libssl-dev ca-certificates libasound2 wget # See https://docs.microsoft.com/en-us/azure/cognitive-services/speech-service/quickstarts/setup-platform From c058b0c14001f59d0af14ade256ed932b56e7ced Mon Sep 17 00:00:00 2001 From: Lawrence Angrave <4468456+angrave@users.noreply.github.com> Date: Thu, 18 Jan 2024 17:45:14 -0600 Subject: [PATCH 3/9] Update Program.cs --- TestAzureCognitiveServices/Program.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TestAzureCognitiveServices/Program.cs b/TestAzureCognitiveServices/Program.cs index ba74073..1a3606a 100644 --- a/TestAzureCognitiveServices/Program.cs +++ b/TestAzureCognitiveServices/Program.cs @@ -143,7 +143,7 @@ static void Main(string[] args) } async static Task useAzureTranslationAsync() { - var defaultKeys = "996885bc424b4fda9df983c404e7309c,westus"; + var defaultKeys = "916885bc424b4fda9df983c404e7309c,westus"; Console.WriteLine($"Environment variable AZURE_SUBSCRIPTION_KEYS=key,region;..."); var keys = System.Environment.GetEnvironmentVariable("AZURE_SUBSCRIPTION_KEYS") ?? defaultKeys; From 6c08dc4cc7592da563e7e44fcaad0ab0135b6b60 Mon Sep 17 00:00:00 2001 From: Lawrence Angrave Date: Thu, 18 Jan 2024 17:53:01 -0600 Subject: [PATCH 4/9] M1 Mac Support --- TaskEngine.Dockerfile | 14 ++++++------- TestAzureCognitiveServices/.dockerignore | 4 +++- .../TestAzure.Dockerfile | 19 ++++++++---------- .../TestAzureCognitiveServices.csproj | 2 +- TestAzureCognitiveServices/install-libssl1.sh | 17 ++++++++++++++++ install-speech-hack-libssl1.sh | 20 +++++++++++++++++++ 6 files changed, 56 insertions(+), 20 deletions(-) create mode 100755 TestAzureCognitiveServices/install-libssl1.sh create mode 100755 install-speech-hack-libssl1.sh diff --git a/TaskEngine.Dockerfile b/TaskEngine.Dockerfile index ce64dea..6ff8d94 100644 --- a/TaskEngine.Dockerfile +++ b/TaskEngine.Dockerfile @@ -20,7 +20,10 @@ COPY ./TaskEngine ./TaskEngine WORKDIR /src/TaskEngine RUN dotnet publish TaskEngine.csproj -c Release -o /app --no-restore -FROM mcr.microsoft.com/dotnet/aspnet:8.0-bookworm-slim-amd64 as publish_base +#FROM mcr.microsoft.com/dotnet/aspnet:8.0-bookworm-slim as publish_base +FROM mcr.microsoft.com/dotnet/aspnet:8.0 as publish_base +# https://hub.docker.com/_/microsoft-dotnet-aspnet/ + # force AMD64 build here: the ssl1.1.1 workaround below assumes amd64 # Install prerequisites for Azure Speech Services: build-essential libssl-dev ca-certificates libasound2 wget # See https://docs.microsoft.com/en-us/azure/cognitive-services/speech-service/quickstarts/setup-platform @@ -30,12 +33,9 @@ apt-get install -y netcat-traditional && apt-get -q update # Microsoft 8.0 issue: https://github.com/Azure-Samples/cognitive-services-speech-sdk/issues/2204 # This will install OpenSSL 1.1.1 because it is needed by the Speech SDK. -RUN \ -wget http://security.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2.20_amd64.deb && \ -wget http://security.ubuntu.com/ubuntu/pool/main/o/openssl/libssl-dev_1.1.1f-1ubuntu2.20_amd64.deb && \ -dpkg -i libssl1.1_1.1.1f-1ubuntu2.20_amd64.deb && \ -dpkg -i libssl-dev_1.1.1f-1ubuntu2.20_amd64.deb && \ -rm libssl1.1_1.1.1f-1ubuntu2.20_amd64.deb libssl-dev_1.1.1f-1ubuntu2.20_amd64.deb +# RUN ARCH=$(dpkg --print-architecture) +COPY ./install-speech-hack-libssl1.sh / +RUN /install-speech-hack-libssl1.sh FROM publish_base as publish diff --git a/TestAzureCognitiveServices/.dockerignore b/TestAzureCognitiveServices/.dockerignore index fd4aebb..0ed6b58 100644 --- a/TestAzureCognitiveServices/.dockerignore +++ b/TestAzureCognitiveServices/.dockerignore @@ -7,4 +7,6 @@ **/*.*proj.user **/charts **/bin -**/obj \ No newline at end of file +**/obj +*Dockerfile +*cproj diff --git a/TestAzureCognitiveServices/TestAzure.Dockerfile b/TestAzureCognitiveServices/TestAzure.Dockerfile index f83cc64..591a27f 100644 --- a/TestAzureCognitiveServices/TestAzure.Dockerfile +++ b/TestAzureCognitiveServices/TestAzure.Dockerfile @@ -6,7 +6,11 @@ COPY . . RUN dotnet restore ./TestAzureCognitiveServices.csproj RUN dotnet publish ./TestAzureCognitiveServices.csproj -c Release -o /app --no-restore -FROM mcr.microsoft.com/dotnet/aspnet:8.0-bookworm-slim as publish_base1 +#FROM mcr.microsoft.com/dotnet/aspnet:8.0-bookworm-slim as publish_base1 +# FROM mcr.microsoft.com/dotnet/aspnet:8.0-bookworm-slim-arm64v8 as publish_base1 +FROM mcr.microsoft.com/dotnet/aspnet:8.0 as publish_base1 + +#COPY ./Program.cs / # Grrr AzureServices does not work on dotnet8 on Debian 12 because it wont link to libssl3 - fix below is needed for short-term @@ -17,16 +21,9 @@ RUN apt-get -y install build-essential libssl-dev libasound2 wget # Microsoft 8.0 issue: https://github.com/Azure-Samples/cognitive-services-speech-sdk/issues/2204 # This will install OpenSSL 1.1.1 because it is needed by the Speech SDK. -RUN \ -wget http://security.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2.20_amd64.deb && \ -wget http://security.ubuntu.com/ubuntu/pool/main/o/openssl/libssl-dev_1.1.1f-1ubuntu2.20_amd64.deb && \ -dpkg -i libssl1.1_1.1.1f-1ubuntu2.20_amd64.deb && \ -dpkg -i libssl-dev_1.1.1f-1ubuntu2.20_amd64.deb && \ -rm libssl1.1_1.1.1f-1ubuntu2.20_amd64.deb libssl-dev_1.1.1f-1ubuntu2.20_amd64.deb - - - - +#RUN ARCH=$(dpkg --print-architecture) +COPY ./install-libssl1.sh / +RUN /install-libssl1.sh FROM publish_base1 as publish1 WORKDIR /app diff --git a/TestAzureCognitiveServices/TestAzureCognitiveServices.csproj b/TestAzureCognitiveServices/TestAzureCognitiveServices.csproj index 4222922..35e56ba 100644 --- a/TestAzureCognitiveServices/TestAzureCognitiveServices.csproj +++ b/TestAzureCognitiveServices/TestAzureCognitiveServices.csproj @@ -6,7 +6,7 @@ - + diff --git a/TestAzureCognitiveServices/install-libssl1.sh b/TestAzureCognitiveServices/install-libssl1.sh new file mode 100755 index 0000000..57641c6 --- /dev/null +++ b/TestAzureCognitiveServices/install-libssl1.sh @@ -0,0 +1,17 @@ +#!/bin/sh +# Temporary 2024 Hack for MS SpeechSDK to run on dotnet8 +ARCH=$(dpkg --print-architecture) + +if [ "$ARCH" = "arm64" ] ; then + BASE="http://ports.ubuntu.com/ubuntu-ports/pool/main/o/openssl/" +else + BASE="http://security.ubuntu.com/ubuntu/pool/main/o/openssl/" +fi + +wget $BASE/libssl1.1_1.1.1f-1ubuntu2.20_${ARCH}.deb +wget $BASE/libssl-dev_1.1.1f-1ubuntu2.20_${ARCH}.deb +dpkg -i libssl1.1_1.1.1f-1ubuntu2.20_${ARCH}.deb +dpkg -i libssl-dev_1.1.1f-1ubuntu2.20_${ARCH}.deb +rm libssl1.1_1.1.1f-1ubuntu2.20_${ARCH}.deb libssl-dev_1.1.1f-1ubuntu2.20_${ARCH}.deb + + diff --git a/install-speech-hack-libssl1.sh b/install-speech-hack-libssl1.sh new file mode 100755 index 0000000..0af8a78 --- /dev/null +++ b/install-speech-hack-libssl1.sh @@ -0,0 +1,20 @@ +#!/bin/sh + +# Temporary 2024 Hack for MS SpeechSDK to run on dotnet8 +# https://github.com/Azure-Samples/cognitive-services-speech-sdk/issues/2204 + +ARCH=$(dpkg --print-architecture) + +if [ "$ARCH" = "arm64" ] ; then + BASE="http://ports.ubuntu.com/ubuntu-ports/pool/main/o/openssl/" +else + BASE="http://security.ubuntu.com/ubuntu/pool/main/o/openssl/" +fi + +wget $BASE/libssl1.1_1.1.1f-1ubuntu2.20_${ARCH}.deb +wget $BASE/libssl-dev_1.1.1f-1ubuntu2.20_${ARCH}.deb +dpkg -i libssl1.1_1.1.1f-1ubuntu2.20_${ARCH}.deb +dpkg -i libssl-dev_1.1.1f-1ubuntu2.20_${ARCH}.deb +rm libssl1.1_1.1.1f-1ubuntu2.20_${ARCH}.deb libssl-dev_1.1.1f-1ubuntu2.20_${ARCH}.deb + + From 66c00808f8464b991cf8eb9bab2a4c49dba71b98 Mon Sep 17 00:00:00 2001 From: Lawrence Angrave Date: Thu, 18 Jan 2024 17:56:47 -0600 Subject: [PATCH 5/9] Bump SpeechSDK to latest version 1.34.1 --- ClassTranscribeDatabase/ClassTranscribeDatabase.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ClassTranscribeDatabase/ClassTranscribeDatabase.csproj b/ClassTranscribeDatabase/ClassTranscribeDatabase.csproj index 987885a..e43371f 100644 --- a/ClassTranscribeDatabase/ClassTranscribeDatabase.csproj +++ b/ClassTranscribeDatabase/ClassTranscribeDatabase.csproj @@ -39,7 +39,7 @@ - + runtime; build; native; contentfiles; analyzers; buildtransitive From dd9ef3fa5495547ec7d001d87aa743228e6a6b15 Mon Sep 17 00:00:00 2001 From: Lawrence Angrave Date: Thu, 18 Jan 2024 17:57:02 -0600 Subject: [PATCH 6/9] remove async void --- TaskEngine/Tasks/TranscriptionTask.cs | 2 +- .../ControllerTests/UserOfferingsControllerTest.cs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/TaskEngine/Tasks/TranscriptionTask.cs b/TaskEngine/Tasks/TranscriptionTask.cs index 397e3df..8b84056 100644 --- a/TaskEngine/Tasks/TranscriptionTask.cs +++ b/TaskEngine/Tasks/TranscriptionTask.cs @@ -38,7 +38,7 @@ public TranscriptionTask(RabbitMQConnection rabbitMQ, MSTranscriptionService msT _captionQueries = captionQueries; } - private async void buildMockCaptions(string videoId) + private async Task buildMockCaptions(string videoId) { GetLogger().LogInformation($"Building Mock Captions for video {videoId}"); diff --git a/UnitTests/ClassTranscribeServer/ControllerTests/UserOfferingsControllerTest.cs b/UnitTests/ClassTranscribeServer/ControllerTests/UserOfferingsControllerTest.cs index a2d4cd2..3b5fe59 100644 --- a/UnitTests/ClassTranscribeServer/ControllerTests/UserOfferingsControllerTest.cs +++ b/UnitTests/ClassTranscribeServer/ControllerTests/UserOfferingsControllerTest.cs @@ -348,7 +348,7 @@ private void AddUsers(List ids) { } } - private async void AddUserOfferingsOneToOne(string offeringId, List ids, List roles) { + private async Task AddUserOfferingsOneToOne(string offeringId, List ids, List roles) { Assert.Equal(ids.Count(), roles.Count()); for (var i = 0; i < ids.Count(); i++) { @@ -362,7 +362,7 @@ private async void AddUserOfferingsOneToOne(string offeringId, List ids, } } - private async void AddUserOfferingsOneToMany(string offeringId, string id, List roles) { + private async Task AddUserOfferingsOneToMany(string offeringId, string id, List roles) { for (var i = 0; i < roles.Count(); i++) { var postResult = await _controller.PostUserOffering(new UserOfferingDTO From f9085768904682629d4b6b1452b58df666caf7a3 Mon Sep 17 00:00:00 2001 From: Lawrence Angrave Date: Fri, 19 Jan 2024 14:55:45 -0600 Subject: [PATCH 7/9] UpdateRelatedEntitiyLoading --- .../Controllers/MediaController.cs | 4 +++- .../Controllers/PlaylistsController.cs | 9 +++++++++ ClassTranscribeServer/Utils/Authorization.cs | 4 +++- TaskEngine/Tasks/DescribeVideoTask.cs | 15 +++++++++------ TaskEngine/Tasks/DownloadMediaTask.cs | 11 +++++++++-- 5 files changed, 33 insertions(+), 10 deletions(-) diff --git a/ClassTranscribeServer/Controllers/MediaController.cs b/ClassTranscribeServer/Controllers/MediaController.cs index e95ca06..952e29b 100644 --- a/ClassTranscribeServer/Controllers/MediaController.cs +++ b/ClassTranscribeServer/Controllers/MediaController.cs @@ -40,7 +40,9 @@ public MediaController(IAuthorizationService authorizationService, [HttpGet("{id}")] public async Task> GetMedia(string id) { - var media = await _context.Medias.FindAsync(id); + var media = await _context.Medias. + Include(m => m.Video).ThenInclude(v => v.Transcriptions). + Where(m => m.Id == id).FirstAsync(); if (media == null) { diff --git a/ClassTranscribeServer/Controllers/PlaylistsController.cs b/ClassTranscribeServer/Controllers/PlaylistsController.cs index 2a85a2f..118ad68 100644 --- a/ClassTranscribeServer/Controllers/PlaylistsController.cs +++ b/ClassTranscribeServer/Controllers/PlaylistsController.cs @@ -133,6 +133,13 @@ public async Task>> GetPlaylists2(string o } var playLists = await _context.Playlists + .Include(p=>p.Medias).ThenInclude(m=>m.Video).ThenInclude(v=>v.Transcriptions) + .Include(p=>p.Medias).ThenInclude(m=>m.Video).ThenInclude(v=>v.ProcessedVideo1) + .Include(p=>p.Medias).ThenInclude(m=>m.Video).ThenInclude(v=>v.ProcessedVideo2) + .Include(p=>p.Medias).ThenInclude(m=>m.Video).ThenInclude(v=>v.Video1) + .Include(p=>p.Medias).ThenInclude(m=>m.Video).ThenInclude(v=>v.Video2) + .Include(p=>p.Medias).ThenInclude(m=>m.Video).ThenInclude(v=>v.ASLVideo) + .Where(p => p.OfferingId == offeringId) .OrderBy(p => p.Index) .ThenBy(p => p.CreatedAt).ToListAsync(); @@ -216,6 +223,8 @@ public async Task>> SearchForMedia(stri public async Task> GetPlaylist(string id) { var p = await _context.Playlists.FindAsync(id); + // Media are explicitly loaded below, so LoadAsync is not needed + var user = await _userUtils.GetUser(User); if (p == null) { diff --git a/ClassTranscribeServer/Utils/Authorization.cs b/ClassTranscribeServer/Utils/Authorization.cs index 84b540b..254b52d 100644 --- a/ClassTranscribeServer/Utils/Authorization.cs +++ b/ClassTranscribeServer/Utils/Authorization.cs @@ -84,7 +84,9 @@ protected override async Task HandleRequirementAsync(AuthorizationHandlerContext } else if (offering != null && offering.AccessType == AccessTypes.UniversityOnly && user != null) { - var universityId = await _ctDbContext.CourseOfferings.Where(co => co.OfferingId == offering.Id) + var universityId = await _ctDbContext.CourseOfferings + .Include(co=>co.Course).ThenInclude(c=>c.Department) + .Where(co => co.OfferingId == offering.Id) .Select(c => c.Course.Department.UniversityId).FirstAsync(); if (user.UniversityId == universityId) { diff --git a/TaskEngine/Tasks/DescribeVideoTask.cs b/TaskEngine/Tasks/DescribeVideoTask.cs index b23bb98..c7d48f7 100644 --- a/TaskEngine/Tasks/DescribeVideoTask.cs +++ b/TaskEngine/Tasks/DescribeVideoTask.cs @@ -1,12 +1,14 @@ -using ClassTranscribeDatabase; -using ClassTranscribeDatabase.Models; -using ClassTranscribeDatabase.Services;using Microsoft.Extensions.Logging; -using Newtonsoft.Json.Linq; -using System.Collections.Generic; +using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System; using System.Linq; using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.EntityFrameworkCore; +using Newtonsoft.Json.Linq; +using ClassTranscribeDatabase; +using ClassTranscribeDatabase.Models; +using ClassTranscribeDatabase.Services; using static ClassTranscribeDatabase.CommonUtils; @@ -35,13 +37,14 @@ protected async override Task OnConsume(string videoId, TaskParameters taskParam using var _context = CTDbContext.CreateDbContext(); Video video = await _context.Videos.FindAsync(videoId); - + if (!video.HasSceneObjectData()) { GetLogger().LogInformation($"Describe Video {videoId}: Early return - no scene data to process"); return; } TextData td = await _context.TextData.FindAsync(video.SceneObjectDataId); + await _context.Transcriptions.Where(t => t.VideoId == videoId).LoadAsync(); JObject sceneData = td.GetAsJSON() as JObject; JArray scenes = sceneData["Scenes"] as JArray; diff --git a/TaskEngine/Tasks/DownloadMediaTask.cs b/TaskEngine/Tasks/DownloadMediaTask.cs index af9c6f6..c25b352 100644 --- a/TaskEngine/Tasks/DownloadMediaTask.cs +++ b/TaskEngine/Tasks/DownloadMediaTask.cs @@ -52,7 +52,7 @@ protected override async Task OnConsume(string mediaId, TaskParameters taskParam media = await _context.Medias.Where(m => m.Id == mediaId) .Include(m => m.Playlist).FirstAsync(); GetLogger().LogInformation($"Downloading media id=({media.Id}), UniqueMediaIdentifier={media.UniqueMediaIdentifier}"); - subdir = ToCourseOfferingSubDirectory(_context, media); // e.g. "/data/2203-abcd" + subdir = ToCourseOfferingSubDirectory(_context, media.Playlist); // e.g. "/data/2203-abcd" } Video video = new Video(); switch (media.SourceType) @@ -72,7 +72,14 @@ protected override async Task OnConsume(string mediaId, TaskParameters taskParam using (var _context = CTDbContext.CreateDbContext()) { - var latestMedia = await _context.Medias.FindAsync(media.Id); + var latestMedia = await _context.Medias + .Include(m=>m.Video).ThenInclude(v=>v.Video2) + .Include(m=>m.Video).ThenInclude(v=>v.Video1) + .FirstOrDefaultAsync(m => m.Id==media.Id); // Find does not support Include + if(latestMedia == null) { // should never happen... + GetLogger().LogInformation($"Media ({media.Id}): latestMedia == null !?"); + return; + } GetLogger().LogInformation($"Media ({media.Id}): latestMedia.Video == null is {latestMedia.Video == null}"); // Don't add video if there are already videos for the given media. From 284c1a45c561e5585e987714994ed95a401fccbf Mon Sep 17 00:00:00 2001 From: Lawrence Angrave <4468456+angrave@users.noreply.github.com> Date: Fri, 19 Jan 2024 14:58:05 -0600 Subject: [PATCH 8/9] Update Program.cs --- TestAzureCognitiveServices/Program.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TestAzureCognitiveServices/Program.cs b/TestAzureCognitiveServices/Program.cs index 1a3606a..94cbb50 100644 --- a/TestAzureCognitiveServices/Program.cs +++ b/TestAzureCognitiveServices/Program.cs @@ -143,7 +143,7 @@ static void Main(string[] args) } async static Task useAzureTranslationAsync() { - var defaultKeys = "916885bc424b4fda9df983c404e7309c,westus"; + var defaultKeys = "123,westus"; Console.WriteLine($"Environment variable AZURE_SUBSCRIPTION_KEYS=key,region;..."); var keys = System.Environment.GetEnvironmentVariable("AZURE_SUBSCRIPTION_KEYS") ?? defaultKeys; From 7e357189bca2e68881c0d0948e9ea4f4b5b99698 Mon Sep 17 00:00:00 2001 From: Lawrence Angrave Date: Fri, 19 Jan 2024 15:02:21 -0600 Subject: [PATCH 9/9] Return null if missing --- ClassTranscribeServer/Controllers/MediaController.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ClassTranscribeServer/Controllers/MediaController.cs b/ClassTranscribeServer/Controllers/MediaController.cs index 952e29b..b890510 100644 --- a/ClassTranscribeServer/Controllers/MediaController.cs +++ b/ClassTranscribeServer/Controllers/MediaController.cs @@ -42,7 +42,7 @@ public async Task> GetMedia(string id) { var media = await _context.Medias. Include(m => m.Video).ThenInclude(v => v.Transcriptions). - Where(m => m.Id == id).FirstAsync(); + Where(m => m.Id == id).FirstOrDefaultAsync(); if (media == null) {