Implement file compression

.gitignore

@@ -85,6 +85,9 @@ build/
 *.tar
 *.tgz
 
+### Include compressed file for testing GzipCompressUtils
+!EOB-500.ndjson.gz
+
 ### Eclipse
 .checkstyle
 

api/src/main/java/gov/cms/ab2d/api/controller/common/FileDownloadCommon.java

@@ -2,6 +2,8 @@
 
 import gov.cms.ab2d.api.remote.JobClient;
 import gov.cms.ab2d.common.service.PdpClientService;
+import gov.cms.ab2d.common.util.Constants;
+import gov.cms.ab2d.common.util.GzipCompressUtils;
 import gov.cms.ab2d.eventclient.clients.SQSEventClient;
 import gov.cms.ab2d.eventclient.events.ApiResponseEvent;
 import java.io.FileInputStream;
@@ -11,6 +13,7 @@
 import javax.servlet.http.HttpServletResponse;
 import lombok.AllArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import lombok.val;
 import org.apache.commons.io.IOUtils;
 import org.slf4j.MDC;
 import org.springframework.core.io.Resource;
@@ -19,7 +22,8 @@
 import org.springframework.http.ResponseEntity;
 import org.springframework.stereotype.Service;
 
-
+import static gov.cms.ab2d.api.controller.common.FileDownloadCommon.Encoding.GZIP_COMPRESSED;
+import static gov.cms.ab2d.api.controller.common.FileDownloadCommon.Encoding.UNCOMPRESSED;
 import static gov.cms.ab2d.common.util.Constants.FILE_LOG;
 import static gov.cms.ab2d.common.util.Constants.JOB_LOG;
 import static gov.cms.ab2d.common.util.Constants.FHIR_NDJSON_CONTENT_TYPE;
@@ -34,28 +38,96 @@ public class FileDownloadCommon {
     private final SQSEventClient eventLogger;
     private final PdpClientService pdpClientService;
 
+    enum Encoding {
+        UNCOMPRESSED,
+        GZIP_COMPRESSED;
+    }
+
     public ResponseEntity<String> downloadFile(String jobUuid, String filename, HttpServletRequest request, HttpServletResponse response) throws IOException {
         MDC.put(JOB_LOG, jobUuid);
         MDC.put(FILE_LOG, filename);
         log.info("Request submitted to download file");
 
-        Resource downloadResource = jobClient.getResourceForJob(jobUuid, filename,
-                pdpClientService.getCurrentClient().getOrganization());
+        final Resource downloadResource = getDownloadResource(jobUuid, filename);
 
         log.info("Sending " + filename + " file to client");
 
-        String fileDownloadName = downloadResource.getFile().getName();
-
         response.setHeader(HttpHeaders.CONTENT_TYPE, FHIR_NDJSON_CONTENT_TYPE);
-        response.setHeader("Content-Disposition", "inline; swaggerDownload=\"attachment\"; filename=\"" + fileDownloadName + "\"");
 
-        try (OutputStream out = response.getOutputStream(); FileInputStream in = new FileInputStream(downloadResource.getFile())) {
-            IOUtils.copy(in, out);
+        try (OutputStream out = response.getOutputStream();
+            FileInputStream in = new FileInputStream(downloadResource.getFile())) {
+
+            // set headers before writing to response stream
+            final Encoding fileEncoding = getFileEncoding(downloadResource);
+            final Encoding requestedEncoding = getRequestedEncoding(request);
+            if (requestedEncoding == GZIP_COMPRESSED) {
+                response.setHeader("Content-Encoding", Constants.GZIP_ENCODING);
+            }
+            final String fileDownloadName = getDownloadFilename(downloadResource, requestedEncoding);
+            response.setHeader("Content-Disposition", "inline; swaggerDownload=\"attachment\"; filename=\"" + fileDownloadName + "\"");
+
+            // write to response stream, compressing or decompressing file contents depending on 'Accept-Encoding' header
+            if (requestedEncoding == fileEncoding) {
+                IOUtils.copy(in, out);
+            } else if (fileEncoding == GZIP_COMPRESSED && requestedEncoding == UNCOMPRESSED) {
+                GzipCompressUtils.decompress(in, response.getOutputStream());
+            } else if (fileEncoding == UNCOMPRESSED && requestedEncoding == GZIP_COMPRESSED) {
+                GzipCompressUtils.compress(in, response.getOutputStream());
+            }
 
             eventLogger.sendLogs(new ApiResponseEvent(MDC.get(ORGANIZATION), jobUuid, HttpStatus.OK, "File Download",
                     "File " + filename + " was downloaded", (String) request.getAttribute(REQUEST_ID)));
             jobClient.incrementDownload(downloadResource.getFile(), jobUuid);
             return new ResponseEntity<>(null, null, HttpStatus.OK);
         }
     }
+
+    Resource getDownloadResource(String jobUuid, String filename) throws IOException {
+        val organization = pdpClientService.getCurrentClient().getOrganization();
+        try {
+            // look for compressed file
+            return jobClient.getResourceForJob(jobUuid, filename + ".gz", organization);
+        } catch (RuntimeException e) {
+            // look for uncompressed file
+            // allow this exception to be thrown to caller (for consistency with current behavior)
+            return jobClient.getResourceForJob(jobUuid, filename, organization);
+        }
+    }
+
+    static String getDownloadFilename(
+            Resource downloadResource,
+            Encoding requestedEncoding) throws IOException {
+
+        final Encoding fileEncoding = getFileEncoding(downloadResource);
+        final String filename = downloadResource.getFile().getName();
+        if (requestedEncoding == fileEncoding) {
+            return filename;
+        } else if (fileEncoding == GZIP_COMPRESSED && requestedEncoding == UNCOMPRESSED) {
+            return filename.replace(".gz", "");
+        } else {
+            return filename + ".gz";
+        }
+    }
+
+    static Encoding getFileEncoding(Resource resource) throws IOException {
+        if (resource.getFile().getName().endsWith(".gz")) {
+            return GZIP_COMPRESSED;
+        }
+        return UNCOMPRESSED;
+    }
+
+    // determine optional encoding requested by user, defaulting to uncompressed if not provided
+    static Encoding getRequestedEncoding(HttpServletRequest request) {
+        val values = request.getHeaders("Accept-Encoding");
+        if (values != null) {
+            while (values.hasMoreElements()) {
+                val header = values.nextElement();
+                if (header.trim().equalsIgnoreCase(Constants.GZIP_ENCODING)) {
+                    return GZIP_COMPRESSED;
+                }
+            }
+        }
+
+        return UNCOMPRESSED;
+    }
 }

api/src/main/java/gov/cms/ab2d/api/controller/common/StatusCommon.java

@@ -20,6 +20,7 @@
 import java.util.List;
 import javax.servlet.http.HttpServletRequest;
 import lombok.extern.slf4j.Slf4j;
+import lombok.val;
 import org.slf4j.MDC;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.http.HttpHeaders;
@@ -149,8 +150,17 @@ protected ResponseEntity<OpenAPIConfig.OperationOutcome> getCanceledResponse(Job
         return new ResponseEntity<OpenAPIConfig.OperationOutcome>(outcome, responseHeaders, HttpStatus.NOT_FOUND);
     }
 
-    private String getUrlPath(String jobUuid, String filePath, HttpServletRequest request, String apiPrefix) {
-        return Common.getUrl(apiPrefix + FHIR_PREFIX + "/Job/" + jobUuid + "/file/" + filePath, request);
+    protected String getUrlPath(String jobUuid, String filePath, HttpServletRequest request, String apiPrefix) {
+        val filePathWithoutGzExtension = removeGzFileExtension(filePath);
+        return Common.getUrl(apiPrefix + FHIR_PREFIX + "/Job/" + jobUuid + "/file/" + filePathWithoutGzExtension, request);
+    }
+
+    // job output files are now stored in gzip format - remove '.gz' extension before building file download URL for backwards compatibility
+    protected String removeGzFileExtension(String filePath) {
+        val index = filePath.lastIndexOf(".gz");
+        return (index == -1)
+                ? filePath
+                : filePath.substring(0, index);
     }
 
     private List<JobCompletedResponse.FileMetadata> generateValueOutputs(JobOutput o) {

api/src/test/java/gov/cms/ab2d/api/controller/common/FileDownloadCommonTest.java

@@ -0,0 +1,78 @@
+package gov.cms.ab2d.api.controller.common;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+import org.springframework.core.io.Resource;
+
+import javax.servlet.http.HttpServletRequest;
+import java.io.File;
+import java.io.IOException;
+
+import static gov.cms.ab2d.api.controller.common.FileDownloadCommon.Encoding.*;
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.Mockito.when;
+import static java.util.Collections.enumeration;
+import static java.util.Arrays.asList;
+
+@ExtendWith(MockitoExtension.class)
+class FileDownloadCommonTest {
+
+    @Mock
+    Resource downloadResource;
+
+    @Mock
+    HttpServletRequest request;
+
+    @Test
+    void download_filename_matches_requested_encoding() throws Exception {
+        when(downloadResource.getFile()).thenReturn(new File("/mnt/efs/xyz/test_1.ndjson"));
+        assertEquals("test_1.ndjson.gz", FileDownloadCommon.getDownloadFilename(downloadResource, GZIP_COMPRESSED));
+        assertEquals("test_1.ndjson.gz", FileDownloadCommon.getDownloadFilename(downloadResource, GZIP_COMPRESSED));
+
+        when(downloadResource.getFile()).thenReturn(new File("/mnt/efs/xyz/test_1.ndjson.gz"));
+        assertEquals("test_1.ndjson", FileDownloadCommon.getDownloadFilename(downloadResource, UNCOMPRESSED));
+        assertEquals("test_1.ndjson", FileDownloadCommon.getDownloadFilename(downloadResource, UNCOMPRESSED));
+    }
+
+    @Test
+    void test_encoding_by_file_extension() throws IOException {
+        when(downloadResource.getFile()).thenReturn(new File("/mnt/efs/xyz/test_1.ndjson"));
+        assertEquals(FileDownloadCommon.getFileEncoding(downloadResource), UNCOMPRESSED);
+
+        when(downloadResource.getFile()).thenReturn(new File("/mnt/efs/xyz/test_1.txt"));
+        assertEquals(FileDownloadCommon.getFileEncoding(downloadResource), UNCOMPRESSED);
+
+        when(downloadResource.getFile()).thenReturn(new File("/mnt/efs/xyz/test_1.ndjson.gz"));
+        assertEquals(FileDownloadCommon.getFileEncoding(downloadResource), GZIP_COMPRESSED);
+    }
+
+    @Test
+    void test_accept_encoding_values() {
+        when(request.getHeaders("Accept-Encoding")).thenReturn(null);
+        assertEquals(UNCOMPRESSED, FileDownloadCommon.getRequestedEncoding(request));
+
+        when(request.getHeaders("Accept-Encoding")).thenReturn(enumeration(asList("")));
+        assertEquals(UNCOMPRESSED, FileDownloadCommon.getRequestedEncoding(request));
+
+        when(request.getHeaders("Accept-Encoding")).thenReturn(enumeration(asList()));
+        assertEquals(UNCOMPRESSED, FileDownloadCommon.getRequestedEncoding(request));
+
+        when(request.getHeaders("Accept-Encoding")).thenReturn(enumeration(asList("gzip2")));
+        assertEquals(UNCOMPRESSED, FileDownloadCommon.getRequestedEncoding(request));
+
+        when(request.getHeaders("Accept-Encoding")).thenReturn(enumeration(asList("gzip")));
+        assertEquals(GZIP_COMPRESSED, FileDownloadCommon.getRequestedEncoding(request));
+
+        when(request.getHeaders("Accept-Encoding")).thenReturn(enumeration(asList("GZIP")));
+        assertEquals(GZIP_COMPRESSED, FileDownloadCommon.getRequestedEncoding(request));
+
+        when(request.getHeaders("Accept-Encoding")).thenReturn(enumeration(asList("test", "gzip")));
+        assertEquals(GZIP_COMPRESSED, FileDownloadCommon.getRequestedEncoding(request));
+
+        when(request.getHeaders("Accept-Encoding")).thenReturn(enumeration(asList("gzip ", "deflate ", "br ")));
+        assertEquals(GZIP_COMPRESSED, FileDownloadCommon.getRequestedEncoding(request));
+    }
+
+}

api/src/test/java/gov/cms/ab2d/api/controller/common/StatusCommonTest.java

@@ -1,7 +1,6 @@
 package gov.cms.ab2d.api.controller.common;
 
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.*;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyBoolean;
 import static org.mockito.ArgumentMatchers.anyInt;
@@ -12,6 +11,8 @@
 
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
 import org.springframework.mock.web.MockHttpServletRequest;
 
 import gov.cms.ab2d.api.controller.JobProcessingException;
@@ -23,6 +24,8 @@
 import gov.cms.ab2d.eventclient.clients.SQSEventClient;
 import gov.cms.ab2d.job.dto.JobPollResult;
 import gov.cms.ab2d.job.model.JobStatus;
+import org.springframework.web.context.request.RequestContextHolder;
+import org.springframework.web.context.request.ServletRequestAttributes;
 
 class StatusCommonTest {
 
@@ -49,6 +52,10 @@ void beforeEach() {
     statusCommon = new StatusCommon(pdpClientService, jobClient, eventLogger, 0);
 
     req = new MockHttpServletRequest();
+    RequestContextHolder.setRequestAttributes(new ServletRequestAttributes(req));
+    req.setScheme("http");
+    req.setServerName("localhost");
+    req.setServerPort(8080);
   }
 
   @Test
@@ -127,4 +134,36 @@ void testGetJobCanceledResponse() {
     );
   }
 
+  @ParameterizedTest
+  @ValueSource(strings = {
+        "Z0000_0001.ndjson.gz",
+        "Z0000_0001.ndjson"
+  })
+  void testGetUrlPathData(String file) {
+    assertEquals(
+            "http://localhost:8080/v1/fhir/Job/1234/file/Z0000_0001.ndjson",
+            statusCommon.getUrlPath("1234", file, req, "v1")
+    );
+  }
+
+  @ParameterizedTest
+  @ValueSource(strings = {
+          "Z0000_0001_error.ndjson.gz",
+          "Z0000_0001_error.ndjson"
+  })
+  void testGetUrlPathError(String file) {
+    assertEquals(
+            "http://localhost:8080/v1/fhir/Job/1234/file/Z0000_0001_error.ndjson",
+            statusCommon.getUrlPath("1234", file, req, "v1")
+    );
+  }
+
+  @Test
+  void testRemoveGzFileExtension() {
+    assertEquals("file.ndjson", statusCommon.removeGzFileExtension("file.ndjson.gz"));
+    assertEquals("file.ndjson", statusCommon.removeGzFileExtension("file.ndjson"));
+  }
+
+
+
 }