Add/reading Sharepoint files

framework/src/main/groovy/ars/rockycube/GenericUtilities.groovy

@@ -199,27 +199,38 @@ class GenericUtilities {
 
 
     public static HashMap extractStatistics(Object input){
-        def len = length(input)
+        // easy for the array
+        if (input instanceof ArrayList) return [rows: length(input)]
 
+        // specialities supported
+        // 1. if the map contains formData, calculate it on the `formData`key
         try {
-            if (input instanceof ArrayList) return [rows: len]
+            def stats = [:]
 
             switch (input.getClass()) {
                 case LinkedHashMap.class:
-                    def stats = (input as LinkedHashMap).get('stats', [:]) as HashMap
-                    stats.put('rows', len)
+                    stats.put('rows', mapSize(input as HashMap))
                     return stats
                 case HashMap.class:
-                    def stats = (input as HashMap).get('stats', [:]) as HashMap
-                    stats.put('rows', len)
+                    stats.put('rows', mapSize(input as HashMap))
                     return stats
                 case LazyMap.class:
-                    def stats = new HashMap<>((input as LazyMap).get('stats', [:]) as LazyMap)
-                    stats.put('rows', len)
+                    stats.put('rows', mapSize(input as LazyMap))
                     return stats
             }
         } catch (Exception ignored){}
-        return [rows: len]
+        return [rows: -1]
+    }
+
+    private static long mapSize(Map input)
+    {
+        def len = 0
+        if (input.containsKey('formData')) {
+            len = length(input['formData'])
+        } else {
+            len = length(input)
+        }
+        return len
     }
 
     private static boolean isFirstCharBracketOrBrace(InputStream is) {

framework/src/main/groovy/ars/rockycube/endpoint/EndpointServiceHandler.groovy

@@ -1472,25 +1472,13 @@ class EndpointServiceHandler {
                 .jsonObject(payload)
                 .withRequestFactory(customTimeoutReqFactory)
 
-        // execute
-        RestClient.RestResponse restResponse = restClient.call()
-
-        // check status code
-        if (restResponse.statusCode != 200) {
-            def errMessage = "Response with status ${restResponse.statusCode} returned: ${restResponse.reasonPhrase}"
-            // display more info depending on what is being returned
-            if (restResponse.headers().containsKey('x-exception-detail'))
-            {
-                errMessage = restResponse.headerFirst('x-exception-detail')
-            }
-            throw new EndpointException(errMessage)
-        }
+        def resp = handlePyCalcResponse(ec, restClient)
 
         // must handle all states of the response
-        def rsp = (HashMap) restResponse.jsonObject()
+        def rsp = (HashMap) resp.jsonObject()
 
         // debug what has come out of the processing
-        if (debug) debugFile(ec, processingId, sessionId, "c-h-process-items-result.json", restResponse.jsonObject())
+        if (debug) debugFile(ec, processingId, sessionId, "c-h-process-items-result.json", resp.jsonObject())
 
         // use callback to check/modify response
         if (cbCheckData) {
@@ -1503,6 +1491,26 @@ class EndpointServiceHandler {
         return rsp
     }
 
+    private static RestClient.RestResponse handlePyCalcResponse(
+            ExecutionContext ec,
+            RestClient rc)
+    {
+        // execute
+        RestClient.RestResponse restResponse = rc.call()
+
+        // check status code
+        if (restResponse.statusCode != 200) {
+            def errMessage = "Response with status ${restResponse.statusCode} returned: ${restResponse.reasonPhrase}"
+            // display more info depending on what is being returned
+            if (restResponse.headers().containsKey('x-exception-detail'))
+            {
+                errMessage = restResponse.headerFirst('x-exception-detail')
+            }
+            throw new EndpointException(errMessage)
+        }
+        return restResponse
+    }
+
     /**
      * This method processes items into list, ready for vizualization
      * @param ec
@@ -1546,19 +1554,44 @@ class EndpointServiceHandler {
                                 args: pyCalcArgs
                         ]
                 )
-        RestClient.RestResponse restResponse = restClient.call()
+        def resp = handlePyCalcResponse(ec, restClient)
 
-        // check status code
-        if (restResponse.statusCode != 200) {
-            if (debug) debugFile(ec, null, null, "vizualize-items-exception.${identity}", restResponse.reasonPhrase)
+        HashMap response = resp.jsonObject() as HashMap
+        return response['data']
+    }
 
-            logger.error("Error in response from pyCalc [${restResponse.reasonPhrase}] for session [${identity}]")
-            throw new EndpointException("Response with status ${restResponse.statusCode} returned: ${restResponse.reasonPhrase}")
-        }
+    /**
+     * Fetch file from Sharepoint
+     * @param ec
+     * @param credentials - shall be used to initialize the connection to Sharepoint (e.g. tenantId, clientId, clientSecret)
+     * @param location - where is the file located
+     * @param contentType - JSON?
+     * @return
+     */
+    public static byte[] fetchFileFromSharepoint(
+            ExecutionContext ec,
+            HashMap credentials,
+            HashMap location)
+    {
+        def pycalcHost = System.properties.get("py.server.host")
+        if (!pycalcHost) throw new EndpointException("PY-CALC server host not defined")
 
-        if (debug) debugFile(ec, null, null, "vizualize-items-after-calc.${identity}", restResponse.jsonObject())
+        // data prep
+        def payload = [credentials: credentials, location:location]
+        def customTimeoutReqFactory = new RestClient.SimpleRequestFactory()
 
-        HashMap response = restResponse.jsonObject() as HashMap
-        return response['data']
+        RestClient restClient = ec.service.rest().method(RestClient.POST)
+                .uri("${pycalcHost}/api/v1/utility/sharepoint/fetch-bytes")
+                .timeout(480)
+                .retry(2, 10)
+                .maxResponseSize(50 * 1024 * 1024)
+                .jsonObject(payload)
+                .withRequestFactory(customTimeoutReqFactory)
+
+        // execute
+        def resp = handlePyCalcResponse(ec, restClient)
+
+        // return bytes
+        return resp.bytes()
     }
 }

framework/src/test/groovy/UtilsTests.groovy

@@ -1,3 +1,4 @@
+import ars.rockycube.GenericUtilities
 import com.google.gson.Gson
 import ars.rockycube.util.CollectionUtils
 import com.google.gson.GsonBuilder
@@ -428,4 +429,22 @@ class UtilsTests extends Specification {
 
         assert true
     }
+
+    /**
+     * Unit-testing calculation of statistics chunks
+     */
+    def test_statistics_calculation(){
+        when:
+
+        TestUtilities.testSingleFile((String[]) ['Utils', "stats-calculation", "expected-stats-calc.json"], { Object processed, Object expected, Integer idx ->
+            def incoming = TestUtilities.loadTestResourceJs(processed['filename'])
+            def result = GenericUtilities.extractStatistics(incoming)
+
+            assert result == expected['result']
+        })
+
+        then:
+
+        assert true
+    }
 }

framework/src/test/resources/Utils/stats-calculation/expected-stats-calc.json

@@ -0,0 +1,10 @@
+[
+  [
+    {
+      "filename": "Utils/stats-calculation/proc-output/nop.nos.calc-stats-input-after.dCB2xWeu.json"
+    },
+    {
+      "result": {"rows": 4}
+    }
+  ]
+]

framework/src/test/resources/Utils/stats-calculation/proc-output/nop.nos.calc-stats-input-after.dCB2xWeu.json

@@ -0,0 +1,23 @@
+{
+    "formData": [
+        {
+            "name": "Elizabeth",
+            "_entity": "sample-list"
+        },
+        {
+            "name": "Paul",
+            "_entity": "sample-list"
+        },
+        {
+            "name": "Lucy",
+            "_entity": "sample-list"
+        },
+        {
+            "name": "Eve",
+            "_entity": "sample-list"
+        }
+    ],
+    "layers": [
+
+    ]
+}

moqui-util/src/main/groovy/ars/rockycube/util/TestUtilities.groovy

@@ -375,6 +375,18 @@ public class TestUtilities {
         }
     }
 
+    /**
+     * Method that provides information on whether a directory exists
+     * @param resDir
+     * @return
+     */
+    static boolean checkDirectoryExists(String[] resDir) {
+        String[] checkDir = setResourcePath(resDir)
+        Path dir = Paths.get(FileUtils.getFile(checkDir).absolutePath)
+
+        return Files.exists(dir) && Files.isDirectory(dir)
+    }
+
     /**
      * Find names of all files inside a directory
      * @param resDir