Merge pull request #216 from nulib/4735-audio-download

Implement audio download

lambdas/package.json

@@ -11,6 +11,10 @@
     "@aws-sdk/client-mediaconvert": "^3.410.0",
     "@aws-sdk/client-s3": "^3.410.0",
     "@aws-sdk/s3-request-presigner": "^3.410.0",
-    "@aws-sdk/signature-v4": "^3.130.0"
+    "@aws-sdk/signature-v4": "^3.130.0",
+    "aws-sdk": "^2.1640.0"
+  },
+  "dependencies": {
+    "fluent-ffmpeg": "2.1.2"
   }
 }

lambdas/start-audio-transcode.js

@@ -0,0 +1,41 @@
+const AWS = require('aws-sdk');
+const ffmpeg = require('fluent-ffmpeg');
+const stream = require('stream');
+
+module.exports.handler = async (event) => {
+
+  const s3 = new AWS.S3();
+  const url = event.streamingUrl;
+  const referer = event.referer || null;
+  const inputOptions = referer ? ["-referer", referer] : [];
+  const bucket = event.destinationBucket;
+  const key = event.destinationKey;
+  const pass = new stream.PassThrough();
+
+  return new Promise((resolve, reject) => {
+    ffmpeg()
+      .input(url)
+      .inputOptions(inputOptions)
+      .format('mp3')
+      .output(pass, { end: true })
+      .on('start', () => {
+        s3.upload({
+          Bucket: bucket,
+          Key: key,
+          Body: pass,
+        }, (error, _data) => {
+          if (error) {
+            console.error('upload failed', error);
+            reject(error);
+          } else {
+            resolve({ success: true });
+          }
+        });
+      })
+      .on('error', (error) => {
+        console.error('ffmpeg error', error);
+        reject(error);
+      })
+      .run();
+  });
+}

node/src/handlers/get-file-set-download.js

@@ -18,6 +18,7 @@ const path = require("path");
 exports.handler = wrap(async (event) => {
   const id = event.pathParameters.id;
   const email = event.queryStringParameters?.email;
+  const referer = event.headers?.referer;
 
   const allowPrivate =
     event.userToken.isSuperUser() ||
@@ -32,14 +33,14 @@ exports.handler = wrap(async (event) => {
 
   if (esResponse.statusCode == "200") {
     const doc = JSON.parse(esResponse.body);
-    if (isVideoDownload(doc)) {
+    if (isAVDownload(doc)) {
       if (!email) {
         return invalidRequest(400, "Query string must include email address");
       }
       if (!event.userToken.isSuperUser()) {
         return invalidRequest(401, "Unauthorized");
       }
-      return await processAVDownload(doc, email);
+      return await processAVDownload(doc, email, referer);
     } else if (isImageDownload(doc)) {
       return await IIIFImageRequest(doc);
     } else if (isAltFileDownload(doc)) {
@@ -70,14 +71,12 @@ function isAltFileDownload(doc) {
   );
 }
 
-function isVideoDownload(doc) {
-  // Note - audio is not currently implemented due to an issue with AWS
-  // & MediaConvert and our .m3u8 files
+function isAVDownload(doc) {
   return (
     doc.found &&
     doc._source.role === "Access" &&
     doc._source.mime_type != null &&
-    ["video"].includes(doc._source.mime_type.split("/")[0]) &&
+    ["audio", "video"].includes(doc._source.mime_type.split("/")[0]) &&
     doc._source.streaming_url != null
   );
 }
@@ -91,6 +90,10 @@ function isImageDownload(doc) {
   );
 }
 
+function isAudio(doc) {
+  return ["audio"].includes(doc._source.mime_type.split("/")[0]);
+}
+
 function derivativeKey(doc) {
   const id = doc._id;
   let prefix =
@@ -169,7 +172,7 @@ const IIIFImageRequest = async (doc) => {
   };
 };
 
-async function processAVDownload(doc, email) {
+async function processAVDownload(doc, email, referer) {
   const stepFunctionConfig = process.env.STEP_FUNCTION_ENDPOINT
     ? { endpoint: process.env.STEP_FUNCTION_ENDPOINT }
     : {};
@@ -184,26 +187,37 @@ async function processAVDownload(doc, email) {
   const fileSetLabel = fileSet.label;
   const workId = fileSet.work_id;
   const fileType = fileSet.mime_type.split("/")[0];
-  const destinationKey = `av-downloads/${fileSetId}.mp4`; //TODO - account for audio
-  const destinationLocation = `s3://${destinationBucket}/av-downloads/${fileSetId}`; // TODO - account for audio
-  const settings = videoTranscodeSettings(sourceLocation, destinationLocation); // TODO - account for audio
+  const destinationKey = isAudio(doc)
+    ? `av-downloads/${fileSetId}.mp3`
+    : `av-downloads/${fileSetId}.mp4`;
+  const destinationLocation = `s3://${destinationBucket}/av-downloads/${fileSetId}`;
+  const settings = isAudio(doc)
+    ? {}
+    : videoTranscodeSettings(sourceLocation, destinationLocation);
+  const filename = isAudio(doc) ? `${fileSetId}.mp3` : `${fileSetId}.mp4`;
 
   var params = {
     stateMachineArn: process.env.AV_DOWNLOAD_STATE_MACHINE_ARN,
     input: JSON.stringify({
       configuration: {
+        startAudioTranscodeFunction: process.env.START_AUDIO_TRANSCODE_FUNCTION,
         startTranscodeFunction: process.env.START_TRANSCODE_FUNCTION,
         transcodeStatusFunction: process.env.TRANSCODE_STATUS_FUNCTION,
         getDownloadLinkFunction: process.env.GET_DOWNLOAD_LINK_FUNCTION,
         sendTemplatedEmailFunction: process.env.SEND_TEMPLATED_EMAIL_FUNCTION,
       },
       transcodeInput: {
         settings: settings,
+        type: fileType,
+        streamingUrl: fileSet.streaming_url,
+        referer: referer,
+        destinationBucket: destinationBucket,
+        destinationKey: destinationKey,
       },
       presignedUrlInput: {
         bucket: destinationBucket,
         key: destinationKey,
-        disposition: `${fileSetId}.mp4`,
+        disposition: filename,
       },
       sendEmailInput: {
         to: email,

node/test/integration/get-file-set-download.test.js

@@ -32,30 +32,43 @@ describe("Download file set", () => {
       expect(result.statusCode).to.eq(401);
     });
 
-    it("returns an error if the mime-type is audio/*", async () => {
+    it("returns unauthorized for an audio without a superuser token", async () => {
       mock
         .get("/dc-v2-file-set/_doc/1234")
         .reply(200, helpers.testFixture("mocks/fileset-audio-1234.json"));
 
+      const event = helpers
+        .mockEvent("GET", "/file-sets/{id}/download")
+        .pathParams({ id: 1234 })
+        .queryParams({ email: "example@example.com" })
+        .render();
+      const result = await handler(event);
+      expect(result.statusCode).to.eq(401);
+    });
+
+    it("returns an error for video if it does not contain an email query string parameters", async () => {
+      mock
+        .get("/dc-v2-file-set/_doc/1234")
+        .reply(200, helpers.testFixture("mocks/fileset-video-1234.json"));
+
       const token = new ApiToken().superUser().sign();
 
       const event = helpers
         .mockEvent("GET", "/file-sets/{id}/download")
         .pathParams({ id: 1234 })
-        .queryParams({ email: "example@example.com" })
         .headers({
           Cookie: `${process.env.API_TOKEN_NAME}=${token};`,
         })
         .render();
 
       const result = await handler(event);
-      expect(result.statusCode).to.eq(405);
+      expect(result.statusCode).to.eq(400);
     });
 
-    it("returns an error if it does not contain an email query string parameters", async () => {
+    it("returns an error for audio if it does not contain an email query string parameters", async () => {
       mock
         .get("/dc-v2-file-set/_doc/1234")
-        .reply(200, helpers.testFixture("mocks/fileset-video-1234.json"));
+        .reply(200, helpers.testFixture("mocks/fileset-audio-1234.json"));
 
       const token = new ApiToken().superUser().sign();
 

state_machines/av_download.json

@@ -1,7 +1,29 @@
 {
   "Comment": "HLS stiching and save as file in s3 and email download link",
-  "StartAt": "startTranscode",
+  "StartAt": "audioOrVideo",
   "States": {
+    "audioOrVideo": {
+      "Type": "Choice",
+      "Choices": [
+        {
+          "Variable": "$.transcodeInput.type",
+          "StringEquals": "audio",
+          "Next": "startAudioTranscode"
+        }
+      ],
+      "Default": "startTranscode"
+    },
+    "startAudioTranscode": {
+      "Type": "Task",
+      "Resource": "arn:aws:states:::lambda:invoke",
+      "Parameters": {
+        "Payload.$": "$.transcodeInput",
+        "FunctionName.$": "$.configuration.startAudioTranscodeFunction"
+      },
+      "Next": "getDownloadLink",
+      "InputPath": "$",
+      "ResultPath": "$.audioTranscodeOutput"
+    },
     "startTranscode": {
       "Type": "Task",
       "Resource": "arn:aws:states:::lambda:invoke",

template.yaml

@@ -345,6 +345,7 @@ Resources:
           MEDIA_CONVERT_ROLE_ARN: !Ref MediaConvertRoleArn
           PYRAMID_BUCKET: !Ref PyramidBucket
           REPOSITORY_EMAIL: !Ref RepositoryEmail
+          START_AUDIO_TRANSCODE_FUNCTION: !GetAtt startAudioTranscodeFunction.Arn
           START_TRANSCODE_FUNCTION: !GetAtt startTranscodeFunction.Arn
           TRANSCODE_STATUS_FUNCTION: !GetAtt transcodeStatusFunction.Arn
           GET_DOWNLOAD_LINK_FUNCTION: !GetAtt getDownloadLinkFunction.Arn
@@ -721,6 +722,7 @@ Resources:
             Action:
               - lambda:InvokeFunction
             Resource:
+              - !GetAtt startAudioTranscodeFunction.Arn
               - !GetAtt startTranscodeFunction.Arn
               - !GetAtt transcodeStatusFunction.Arn
               - !GetAtt getDownloadLinkFunction.Arn
@@ -920,6 +922,28 @@ Resources:
               <!--[if gte mso 15]></td></tr></table><![endif]-->
             </body>
           </html>
+  startAudioTranscodeFunction:
+    Type: AWS::Serverless::Function
+    Properties:
+      Runtime: nodejs16.x
+      CodeUri: ./lambdas
+      Handler: start-audio-transcode.handler
+      Description: Performs audio transcode job with ffmpeg
+      Timeout: 900
+      MemorySize: 10240
+      Layers:
+        - !Sub "arn:aws:lambda:us-east-1:${AWS::AccountId}:layer:ffmpeg:9"
+      Policies:
+        Version: 2012-10-17
+        Statement:
+          - Sid: BucketAccess
+            Effect: Allow
+            Action:
+              - s3:PutObject
+            Resource: !Sub "arn:aws:s3:::${MediaConvertDestinationBucket}/*"
+      Environment:
+        Variables:
+          MEDIA_CONVERT_DESTINATION_BUCKET: !Ref MediaConvertDestinationBucket
   startTranscodeFunction:
     Type: AWS::Serverless::Function
     Properties: