Merge pull request #450 from classtranscribe/AddIncludes

New Download logic

ClassTranscribeDatabase/CommonUtils.cs

@@ -167,7 +167,7 @@ public static string GetMediaName(Media media)
             return name;
         }
         public static string ToCourseOfferingSubDirectory(CTDbContext ctx, Entity entity) {
-#nullable enable
+            #nullable enable
             try
             {
                 String? path = GetRelatedCourseOfferingFilePath(ctx, entity);
@@ -185,11 +185,11 @@ public static string ToCourseOfferingSubDirectory(CTDbContext ctx, Entity entity
             // we could still get here if something in the model has been deleted.
         }
 #nullable enable
-        public static string GetRelatedCourseOfferingFilePath(CTDbContext ctx, Entity entity)
+        private static string? GetRelatedCourseOfferingFilePath(CTDbContext ctx, Entity entity)
         {
             return GetRelatedCourseOfferingFilePathAsync(ctx, entity).GetAwaiter().GetResult();
         }
-        public static async Task<string?> GetRelatedCourseOfferingFilePathAsync(CTDbContext ctx, Entity entity)
+        private static async Task<string?> GetRelatedCourseOfferingFilePathAsync(CTDbContext ctx, Entity entity)
         {
             // the only thing that we can trust exists on the given the entity Id
             // Drop recursion... this may reduce the number of SQL calls

ClassTranscribeDatabase/Seed.cs

@@ -51,7 +51,7 @@ public void Seed()
                     string ignore = "57P03"; // "57P03: the database system is starting up";
 
                     if (! ex.Message.Contains(ignore) ){
-                        throw ex;
+                        throw; // throw implicitly to preserve stack trace
                     }
                 }
                 _logger.LogError($"Attempt {attempt} of {maxAttempts}. Cannot connect to Database");

ClassTranscribeDatabase/Services/RabbitMQConnection.cs

@@ -227,7 +227,8 @@ public void PurgeQueue(String queueName)
                 catch (Exception e)
                 {
                     Logger.LogError(e, "Error purging queue {0}", queueName);
-                    throw e;
+                    throw ; // throw implicitly to preserve stack trace
+                    // see https://learn.microsoft.com/en-us/dotnet/fundamentals/code-analysis/quality-rules/ca2200
                 }
 
             }

ClassTranscribeDatabase/Services/Slack.cs

@@ -16,7 +16,9 @@ public class SlackLogger
         private readonly Encoding _encoding = new UTF8Encoding();
         private readonly ILogger _logger;
         AppSettings _appSettings;
-
+        private static string truncate(string s, int maxLength=120) {
+            return s == null ? "<null>" :  s.Length < maxLength -3 ? s : s.Substring(0, maxLength-3) + "...";
+        }
 
         public SlackLogger(IOptions<AppSettings> appSettings, ILogger<SlackLogger> logger)
         {
@@ -25,7 +27,7 @@ public SlackLogger(IOptions<AppSettings> appSettings, ILogger<SlackLogger> logge
 
             // ignore
             string url = _appSettings.SLACK_WEBHOOK_URL?.Trim()??"";
-            if (url.Length > 0 && !url.Contains("<ADD WEBHOOK URL HERE>"))
+            if (url.Length != 0 && url != "<ADD WEBHOOK URL HERE>" && url != "changeme")
             {
                 _uri = new Uri(url);
             }
@@ -97,7 +99,7 @@ public async Task PostMessageAsync(Payload payload)
             catch(Exception ex)
             {
 
-                Console.WriteLine($"EXCEPTION SENDING SLACK MESSAGE TO '{ _uri.OriginalString }' : {ex.GetType().ToString()} : {ex.Message}");
+                Console.WriteLine($"EXCEPTION SENDING SLACK MESSAGE TO '{ _uri.OriginalString }' : {ex.GetType().ToString()} : {truncate(ex.Message)}");
             }
         }
     }

PythonRpcServer/kaltura.py

@@ -62,7 +62,18 @@ class KalturaProvider(MediaProvider):
     def __init__(self):
         self.client, self.ks = self.getClient(
             KALTURA_PARTNER_ID, KALTURA_TOKEN_ID, KATLURA_APP_TOKEN)
-
+    def sanitize(self, s):
+        result = str(s)
+        if KALTURA_TOKEN_ID:
+            result = result.replace(KALTURA_TOKEN_ID,"*TOKEN*")
+        if KATLURA_APP_TOKEN:
+            result = result.replace(KATLURA_APP_TOKEN,"*APP*")
+        return result
+
+    def truncate(self, s,maxLength=120):
+        if len(s) > maxLength-3:
+            return s[0:maxLength-3] + '...'
+        return s
     # Returns the Kaltura SDK client. Only used internally by constructor
     def getClient(self, partnerId, tokenId, appToken):
         config = KalturaConfiguration(partnerId)
@@ -262,7 +273,7 @@ def getPlaylistItems(self, request):
             raise InvalidPlaylistInfoException(
                 "Error during Channel/Playlist processing " + str(e))
         end_time = perf_counter()
-        print(f"getPlaylistItems({request}) returning '{result}'. Processing ({end_time-start_time:.2f}) seconds.")
+        print(f"getPlaylistItems({request}) returning '{self.truncate(self.sanitize(result))}'. Processing ({end_time-start_time:.2f}) seconds.")
         return result
 
     # Main entry point - overrides stub in MediaProvider super class
@@ -275,7 +286,7 @@ def getMedia(self, request):
 
             result =  self.downloadLecture(videoUrl)
             end_time = perf_counter()
-            print(f"getMedia({request}) returning '{result}'. Processing ({end_time-start_time:.2f}) seconds.")
+            print(f"getMedia({request}) returning '{self.truncate(self.sanitize(result))}'. Processing ({end_time-start_time:.2f}) seconds.")
 
             return result
         except Exception as e:

PythonRpcServer/utils.py

@@ -49,11 +49,12 @@ def getRandomString(n):
 # See random.seed - in this file
 
 
-def getTmpFile():
+def getTmpFile(subdir="pythonrpc"):
+    os.mkdir(os.path.join(DATA_DIRECTORY, subdir), exist_ok=True)
     while True:
         # A key space of 34^12 should be sufficient for us...
         filenameSize = 12
-        candidate = os.path.join(DATA_DIRECTORY, getRandomString(filenameSize))
+        candidate = os.path.join(DATA_DIRECTORY, subdir, "tmp_"+getRandomString(filenameSize))
         if not os.path.exists(candidate):
             return candidate
         # We wil never print this, and if we do, no-one will read it.