From 5a65e46c9e9e80254af443037063501d6bba306c Mon Sep 17 00:00:00 2001 From: Michal Rovnanik on WS Date: Fri, 27 Sep 2024 16:17:51 +0200 Subject: [PATCH 1/4] unified PY response handling --- .../endpoint/EndpointServiceHandler.groovy | 83 +++++++++++++------ 1 file changed, 58 insertions(+), 25 deletions(-) diff --git a/framework/src/main/groovy/ars/rockycube/endpoint/EndpointServiceHandler.groovy b/framework/src/main/groovy/ars/rockycube/endpoint/EndpointServiceHandler.groovy index 140944c38..9503e4d0e 100644 --- a/framework/src/main/groovy/ars/rockycube/endpoint/EndpointServiceHandler.groovy +++ b/framework/src/main/groovy/ars/rockycube/endpoint/EndpointServiceHandler.groovy @@ -1472,25 +1472,13 @@ class EndpointServiceHandler { .jsonObject(payload) .withRequestFactory(customTimeoutReqFactory) - // execute - RestClient.RestResponse restResponse = restClient.call() - - // check status code - if (restResponse.statusCode != 200) { - def errMessage = "Response with status ${restResponse.statusCode} returned: ${restResponse.reasonPhrase}" - // display more info depending on what is being returned - if (restResponse.headers().containsKey('x-exception-detail')) - { - errMessage = restResponse.headerFirst('x-exception-detail') - } - throw new EndpointException(errMessage) - } + def resp = handlePyCalcResponse(ec, restClient) // must handle all states of the response - def rsp = (HashMap) restResponse.jsonObject() + def rsp = (HashMap) resp.jsonObject() // debug what has come out of the processing - if (debug) debugFile(ec, processingId, sessionId, "c-h-process-items-result.json", restResponse.jsonObject()) + if (debug) debugFile(ec, processingId, sessionId, "c-h-process-items-result.json", resp.jsonObject()) // use callback to check/modify response if (cbCheckData) { @@ -1503,6 +1491,26 @@ class EndpointServiceHandler { return rsp } + private static RestClient.RestResponse handlePyCalcResponse( + ExecutionContext ec, + RestClient rc) + { + // execute + RestClient.RestResponse restResponse = rc.call() + + // check status code + if (restResponse.statusCode != 200) { + def errMessage = "Response with status ${restResponse.statusCode} returned: ${restResponse.reasonPhrase}" + // display more info depending on what is being returned + if (restResponse.headers().containsKey('x-exception-detail')) + { + errMessage = restResponse.headerFirst('x-exception-detail') + } + throw new EndpointException(errMessage) + } + return restResponse + } + /** * This method processes items into list, ready for vizualization * @param ec @@ -1546,19 +1554,44 @@ class EndpointServiceHandler { args: pyCalcArgs ] ) - RestClient.RestResponse restResponse = restClient.call() + def resp = handlePyCalcResponse(ec, restClient) - // check status code - if (restResponse.statusCode != 200) { - if (debug) debugFile(ec, null, null, "vizualize-items-exception.${identity}", restResponse.reasonPhrase) + HashMap response = resp.jsonObject() as HashMap + return response['data'] + } - logger.error("Error in response from pyCalc [${restResponse.reasonPhrase}] for session [${identity}]") - throw new EndpointException("Response with status ${restResponse.statusCode} returned: ${restResponse.reasonPhrase}") - } + /** + * Fetch file from Sharepoint + * @param ec + * @param credentials - shall be used to initialize the connection to Sharepoint (e.g. tenantId, clientId, clientSecret) + * @param location - where is the file located + * @param contentType - JSON? + * @return + */ + public static byte[] fetchFileFromSharepoint( + ExecutionContext ec, + HashMap credentials, + HashMap location) + { + def pycalcHost = System.properties.get("py.server.host") + if (!pycalcHost) throw new EndpointException("PY-CALC server host not defined") - if (debug) debugFile(ec, null, null, "vizualize-items-after-calc.${identity}", restResponse.jsonObject()) + // data prep + def payload = [credentials: credentials, location:location] + def customTimeoutReqFactory = new RestClient.SimpleRequestFactory() - HashMap response = restResponse.jsonObject() as HashMap - return response['data'] + RestClient restClient = ec.service.rest().method(RestClient.POST) + .uri("${pycalcHost}/api/v1/utility/sharepoint/fetch-bytes") + .timeout(480) + .retry(2, 10) + .maxResponseSize(50 * 1024 * 1024) + .jsonObject(payload) + .withRequestFactory(customTimeoutReqFactory) + + // execute + def resp = handlePyCalcResponse(ec, restClient) + + // return bytes + return resp.bytes() } } \ No newline at end of file From 93d7322311a76368bc25d19f49cf080af64d345c Mon Sep 17 00:00:00 2001 From: Michal Rovnanik on WS Date: Fri, 27 Sep 2024 16:18:26 +0200 Subject: [PATCH 2/4] new method for extracting stats from the incoming object --- .../ars/rockycube/GenericUtilities.groovy | 29 +++++++++++++------ 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/framework/src/main/groovy/ars/rockycube/GenericUtilities.groovy b/framework/src/main/groovy/ars/rockycube/GenericUtilities.groovy index 594279e50..7663446fe 100644 --- a/framework/src/main/groovy/ars/rockycube/GenericUtilities.groovy +++ b/framework/src/main/groovy/ars/rockycube/GenericUtilities.groovy @@ -199,27 +199,38 @@ class GenericUtilities { public static HashMap extractStatistics(Object input){ - def len = length(input) + // easy for the array + if (input instanceof ArrayList) return [rows: length(input)] + // specialities supported + // 1. if the map contains formData, calculate it on the `formData`key try { - if (input instanceof ArrayList) return [rows: len] + def stats = [:] switch (input.getClass()) { case LinkedHashMap.class: - def stats = (input as LinkedHashMap).get('stats', [:]) as HashMap - stats.put('rows', len) + stats.put('rows', mapSize(input as HashMap)) return stats case HashMap.class: - def stats = (input as HashMap).get('stats', [:]) as HashMap - stats.put('rows', len) + stats.put('rows', mapSize(input as HashMap)) return stats case LazyMap.class: - def stats = new HashMap<>((input as LazyMap).get('stats', [:]) as LazyMap) - stats.put('rows', len) + stats.put('rows', mapSize(input as LazyMap)) return stats } } catch (Exception ignored){} - return [rows: len] + return [rows: -1] + } + + private static long mapSize(Map input) + { + def len = 0 + if (input.containsKey('formData')) { + len = length(input['formData']) + } else { + len = length(input) + } + return len } private static boolean isFirstCharBracketOrBrace(InputStream is) { From 6b34a0342b5c7409962f1be6c8eb59b5cfc9926b Mon Sep 17 00:00:00 2001 From: Michal Rovnanik on WS Date: Fri, 27 Sep 2024 16:18:51 +0200 Subject: [PATCH 3/4] new unit test - statistics calculation --- framework/src/test/groovy/UtilsTests.groovy | 19 +++++++++++++++ .../expected-stats-calc.json | 10 ++++++++ ...p.nos.calc-stats-input-after.dCB2xWeu.json | 23 +++++++++++++++++++ 3 files changed, 52 insertions(+) create mode 100644 framework/src/test/resources/Utils/stats-calculation/expected-stats-calc.json create mode 100644 framework/src/test/resources/Utils/stats-calculation/proc-output/nop.nos.calc-stats-input-after.dCB2xWeu.json diff --git a/framework/src/test/groovy/UtilsTests.groovy b/framework/src/test/groovy/UtilsTests.groovy index 712f88f83..e236023d9 100644 --- a/framework/src/test/groovy/UtilsTests.groovy +++ b/framework/src/test/groovy/UtilsTests.groovy @@ -1,3 +1,4 @@ +import ars.rockycube.GenericUtilities import com.google.gson.Gson import ars.rockycube.util.CollectionUtils import com.google.gson.GsonBuilder @@ -428,4 +429,22 @@ class UtilsTests extends Specification { assert true } + + /** + * Unit-testing calculation of statistics chunks + */ + def test_statistics_calculation(){ + when: + + TestUtilities.testSingleFile((String[]) ['Utils', "stats-calculation", "expected-stats-calc.json"], { Object processed, Object expected, Integer idx -> + def incoming = TestUtilities.loadTestResourceJs(processed['filename']) + def result = GenericUtilities.extractStatistics(incoming) + + assert result == expected['result'] + }) + + then: + + assert true + } } diff --git a/framework/src/test/resources/Utils/stats-calculation/expected-stats-calc.json b/framework/src/test/resources/Utils/stats-calculation/expected-stats-calc.json new file mode 100644 index 000000000..9a1defe28 --- /dev/null +++ b/framework/src/test/resources/Utils/stats-calculation/expected-stats-calc.json @@ -0,0 +1,10 @@ +[ + [ + { + "filename": "Utils/stats-calculation/proc-output/nop.nos.calc-stats-input-after.dCB2xWeu.json" + }, + { + "result": {"rows": 4} + } + ] +] \ No newline at end of file diff --git a/framework/src/test/resources/Utils/stats-calculation/proc-output/nop.nos.calc-stats-input-after.dCB2xWeu.json b/framework/src/test/resources/Utils/stats-calculation/proc-output/nop.nos.calc-stats-input-after.dCB2xWeu.json new file mode 100644 index 000000000..108b6207b --- /dev/null +++ b/framework/src/test/resources/Utils/stats-calculation/proc-output/nop.nos.calc-stats-input-after.dCB2xWeu.json @@ -0,0 +1,23 @@ +{ + "formData": [ + { + "name": "Elizabeth", + "_entity": "sample-list" + }, + { + "name": "Paul", + "_entity": "sample-list" + }, + { + "name": "Lucy", + "_entity": "sample-list" + }, + { + "name": "Eve", + "_entity": "sample-list" + } + ], + "layers": [ + + ] +} \ No newline at end of file From b1ad16721edace4ad61eeb6940c01a78a9be0882 Mon Sep 17 00:00:00 2001 From: Michal Rovnanik on WS Date: Tue, 1 Oct 2024 12:33:25 +0200 Subject: [PATCH 4/4] new method that checks if a directory exists, required for integration tests --- .../groovy/ars/rockycube/util/TestUtilities.groovy | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/moqui-util/src/main/groovy/ars/rockycube/util/TestUtilities.groovy b/moqui-util/src/main/groovy/ars/rockycube/util/TestUtilities.groovy index 7dc87b22b..eeb2f49dc 100644 --- a/moqui-util/src/main/groovy/ars/rockycube/util/TestUtilities.groovy +++ b/moqui-util/src/main/groovy/ars/rockycube/util/TestUtilities.groovy @@ -375,6 +375,18 @@ public class TestUtilities { } } + /** + * Method that provides information on whether a directory exists + * @param resDir + * @return + */ + static boolean checkDirectoryExists(String[] resDir) { + String[] checkDir = setResourcePath(resDir) + Path dir = Paths.get(FileUtils.getFile(checkDir).absolutePath) + + return Files.exists(dir) && Files.isDirectory(dir) + } + /** * Find names of all files inside a directory * @param resDir