Merge pull request #101 from MicroFocus/MTiutiu/Encoding_issues - Add…

…ed detection for Charset Encoding

Added detection for Charset Encoding

docs/TestFramework.md

@@ -2,17 +2,22 @@
 
 1. On your current repository (the one containing the code and the tests) create a new branch that will hold the test runner pipeline.
 
-2. Configure the following 3 variables on the project level:
-    1. testsToRun - the value does not matter. The integration will populate this variable with the selected tests in ALM Octane.
+2. Configure the following variables on the project level:
+    #### Mandatory Variables for test runner configuration
+    1. testsToRun - the value does not matter. The integration will populate this variable with the selected tests from ALM Octane.
     2. testRunnerBranch - the value will be the name of the branch created earlier, which holds the test runner pipeline.
     3. testRunnerFramework - the value should be the following:
         * mvnSurefire - For running JUnit/TestNG over Maven Surefire/Failsafe
         * uft - For running UFT-One tests using FTToolsLauncher
-        * custom - For running tests using a custom Framework that can generate Junit results (see our examples [here](CustomTestFrameworkExample.md))
+        * custom - For running tests using a custom Framework that can generate Junit results (see our examples [here](CustomTestFrameworkExample.md))```
+
+    #### Optional Variables
     4. testRunnerCustomPattern - an optional parameter required only if the Framework is custom
-                               - the value will be a json containing a pattern to convert the Automated Test from ALM Octane to the accepted format for the Framework
+                               - the value will be a JSON containing a pattern to convert the Automated Test from ALM Octane to the accepted format for the Framework
+    5. suiteId -  the value does not matter. The integration will populate this variable with the selected Test Suite ID from ALM Octane.
+    6. suiteRunId -  the value does not matter. The integration will populate this variable with the executed Suite Run ID from ALM Octane.
 
-3. In the created branch configure the `.gitlab-ci.yml` file to include the logic for running the tests received in the testsToRun variable.
+4. In the created branch configure the `.gitlab-ci.yml` file to include the logic for running the tests received in the testsToRun variable.
    Example for `mvnSurefire`:
     ```
    image: maven:3.3.9-jdk-8
@@ -54,4 +59,7 @@ As the Test Runner executes a pipeline on the GitLab side, a job entity will als
 Keep in mind that the Test Runner pipeline should only be executed using the Test Runner and not by manually running it from the Pipeline menu.
 
 More about ALM Octane Testing Framework here:
-https://admhelp.microfocus.com/octane/en/latest/Online/Content/AdminGuide/how-setup-testing-integration.htm
+https://admhelp.microfocus.com/octane/en/latest/Online/Content/AdminGuide/how-setup-testing-integration.htm
+
+### Aditional variable configuration
+1. suiteId -  the value does not matter. The integration will populate this variable with the selected tests in ALM Octane.

pom.xml

@@ -5,7 +5,7 @@
 
     <groupId>com.microfocus.octane.ciplugins</groupId>
     <artifactId>octane-gitlab-service</artifactId>
-    <version>23.3.1</version>
+    <version>23.3.2</version>
 
     <name>ALM Octane GitLab CI Service</name>
     <description>
@@ -108,6 +108,12 @@
     </pluginRepositories>
 
     <dependencies>
+        <dependency>
+            <groupId>com.github.albfernandez</groupId>
+            <artifactId>juniversalchardet</artifactId>
+            <version>2.4.0</version>
+        </dependency>
+
 
         <!--        <dependency>-->
         <!--            <groupId>jakarta.activation</groupId>-->
@@ -221,7 +227,7 @@
         <dependency>
             <groupId>org.json</groupId>
             <artifactId>json</artifactId>
-            <version>20230227</version>
+            <version>20231013</version>
         </dependency>
         <dependency>
             <artifactId>integrations-sdk</artifactId>

src/main/java/com/microfocus/octane/gitlab/api/EventListener.java

@@ -83,7 +83,9 @@
 import javax.ws.rs.Produces;
 import javax.ws.rs.core.Response;
 import javax.xml.transform.TransformerConfigurationException;
-import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.math.BigDecimal;
@@ -307,16 +309,19 @@ private void sendCodeCoverage(long projectId, Project project, Job job) throws G
             String octaneJobId = project.getPathWithNamespace().toLowerCase() + "/" + job.getName();
             String octaneBuildId = job.getId().toString();
 
-            try (InputStream artifactsStream = gitLabApi.getJobApi()
-                    .downloadArtifactsFile(projectId, job.getId())) {
-                List<Map.Entry<String, ByteArrayInputStream>> coverageResultFiles =
-                        TestResultsHelper.extractArtifacts(artifactsStream, "glob:" + coverageReportFilePattern);
+            try (InputStream artifactsStream = gitLabApi.getJobApi().downloadArtifactsFile(projectId, job.getId())) {
+                List<File> coverageResultFiles =
+                        TestResultsHelper.extractArtifactsToFiles(artifactsStream, "glob:" + coverageReportFilePattern);
 
                 if (Objects.nonNull(coverageResultFiles) && coverageResultFiles.size() > 0) {
-                    coverageResultFiles.forEach(
-                            coverageFile -> OctaneSDK.getClients().forEach(client -> client.getCoverageService()
-                                    .pushCoverage(octaneJobId, octaneBuildId, CoverageReportType.JACOCOXML,
-                                            coverageFile.getValue())));
+                    coverageResultFiles.forEach(coverageFile -> OctaneSDK.getClients().forEach(client -> {
+                        try {
+                            client.getCoverageService().pushCoverage(octaneJobId, octaneBuildId, CoverageReportType.JACOCOXML,
+                                    new FileInputStream(coverageFile));
+                        } catch (FileNotFoundException e) {
+                            throw new RuntimeException(e);
+                        }
+                    }));
                 }
             } catch (GitLabApiException | IOException exception) {
                 log.error(exception.getMessage());

src/main/java/com/microfocus/octane/gitlab/helpers/EncodingHelper.java

@@ -0,0 +1,18 @@
+package com.microfocus.octane.gitlab.helpers;
+
+import org.mozilla.universalchardet.UniversalDetector;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+
+public class EncodingHelper {
+
+    public static String detectCharset(InputStream is) throws IOException {
+        return UniversalDetector.detectCharset(is);
+    }
+
+    public static String detectCharset(File file) throws IOException {
+        return UniversalDetector.detectCharset(file);
+    }
+}

src/main/java/com/microfocus/octane/gitlab/helpers/TestResultsHelper.java

@@ -39,7 +39,9 @@
 import org.gitlab4j.api.models.Project;
 
 import java.io.*;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.FileSystems;
+import java.nio.file.Files;
 import java.nio.file.PathMatcher;
 import java.nio.file.Paths;
 import java.util.*;
@@ -103,15 +105,16 @@ public static List<File> extractArtifactsToFiles(InputStream inputStream, String
 
                 if (matcher.matches(Paths.get(entry.getName()))) {
                     ByteArrayOutputStream entryStream = new ByteArrayOutputStream();
-
                     try (InputStream zipEntryStream = zipFile.getInputStream(entry)) {
                         StreamHelper.copyStream(zipEntryStream, entryStream);
                     }
 
                     File tempResultFile = File.createTempFile(entry.getName(),".xml");
-                    FileOutputStream f = null;//new FileOutputStream(entry.getName());
-                    IOUtils.copy(new ByteArrayInputStream(entryStream.toByteArray()), tempResultFile);
-
+                    ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(entryStream.toByteArray());
+                    String encoding = EncodingHelper.detectCharset(byteArrayInputStream);
+                    String xml = entryStream.toString(encoding == null ? StandardCharsets.UTF_8.name() : encoding);
+                    xml = xml.trim().replaceFirst("^([\\W]+)<", "<");
+                    Files.write(tempResultFile.toPath(), xml.getBytes(encoding == null ? StandardCharsets.UTF_8.name() : encoding));
                     result.add(tempResultFile);
                 }
             }
@@ -126,7 +129,7 @@ public static List<File> extractArtifactsToFiles(InputStream inputStream, String
         }
     }
 
-    public static List<Map.Entry<String, ByteArrayInputStream>> extractArtifacts(InputStream inputStream, String testResultsFilePattern) {
+    public static List<Map.Entry<String, String>> extractArtifacts(InputStream inputStream, String testResultsFilePattern) {
         PathMatcher matcher = FileSystems.getDefault()
                 .getPathMatcher(testResultsFilePattern);
         File tempFile = null;
@@ -141,7 +144,7 @@ public static List<Map.Entry<String, ByteArrayInputStream>> extractArtifacts(Inp
 
             ZipFile zipFile = new ZipFile(tempFile);
 
-            List<Map.Entry<String, ByteArrayInputStream>> result = new LinkedList<>();
+            List<Map.Entry<String, String>> result = new LinkedList<>();
             Enumeration<? extends ZipEntry> entries = zipFile.entries();
 
             while (entries.hasMoreElements()) {
@@ -153,8 +156,11 @@ public static List<Map.Entry<String, ByteArrayInputStream>> extractArtifacts(Inp
                     try (InputStream zipEntryStream = zipFile.getInputStream(entry)) {
                         StreamHelper.copyStream(zipEntryStream, entryStream);
                     }
-
-                    result.add(Pair.of(entry.getName(), new ByteArrayInputStream(entryStream.toByteArray())));
+                    ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(entryStream.toByteArray());
+                    String encoding = EncodingHelper.detectCharset(byteArrayInputStream);
+                    String xml = entryStream.toString(encoding != null ? encoding : StandardCharsets.UTF_8.name());
+                    xml = xml.trim().replaceFirst("^([\\W]+)<", "<");
+                    result.add(Pair.of(entry.getName(), xml));
                 }
             }
             return result;

src/main/java/com/microfocus/octane/gitlab/testresults/JunitTestResultsProvider.java

@@ -54,9 +54,9 @@
 import javax.xml.transform.TransformerFactory;
 import javax.xml.transform.stream.StreamResult;
 import javax.xml.transform.stream.StreamSource;
-import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.InputStream;
+import java.io.StringReader;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -98,17 +98,16 @@ public  List<TestRun> createAndGetTestList(InputStream artifactFiles){
         if(TestResultsHelper.isFilePatternExist(testResultsFilePattern)){
 
             try {
-                List<Map.Entry<String, ByteArrayInputStream>> artifacts = TestResultsHelper.extractArtifacts(artifactFiles,testResultsFilePattern);
+                List<Map.Entry<String, String>> artifacts = TestResultsHelper.extractArtifacts(artifactFiles,testResultsFilePattern);
                 JAXBContext jaxbContext = JAXBContext.newInstance(Testsuites.class);
-                for (Map.Entry<String, ByteArrayInputStream> artifact : artifacts) {
+                for (Map.Entry<String, String> artifact : artifacts) {
                     try {
                         DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
                         dbf.setFeature("http://xml.org/sax/features/external-general-entities", false);
                         dbf.setFeature("http://xml.org/sax/features/external-parameter-entities", false);
                         DocumentBuilder db = dbf.newDocumentBuilder();
-                        Document doc = db.parse(new InputSource(artifact.getValue()));
+                        Document doc = db.parse(new InputSource(new StringReader(artifact.getValue())));
                         String rootTagName = doc.getDocumentElement().getTagName().toLowerCase();
-                        artifact.getValue().reset();
                         switch (rootTagName) {
                             case "testsuites":
                             case "testsuite":
@@ -117,8 +116,8 @@ public  List<TestRun> createAndGetTestList(InputStream artifactFiles){
                             case "test-run":
                             case "test-results":
                                 ByteArrayOutputStream os = new ByteArrayOutputStream();
-                                nunitTransformer.transform(new StreamSource(artifact.getValue()), new StreamResult(os));
-                                unmarshallAndAddToResults(result, jaxbContext, new ByteArrayInputStream(os.toByteArray()));
+                                nunitTransformer.transform(new StreamSource(new StringReader(artifact.getValue())), new StreamResult(os));
+                                unmarshallAndAddToResults(result, jaxbContext, os.toString());
                                 break;
                             default:
                                 log.error(String.format("Artifact %s: unknown test result format that starts with the <%s> tag", artifact.getKey(), rootTagName));
@@ -140,7 +139,7 @@ public boolean createTestList(Project project, Job job, InputStream artifactFile
         if(TestResultsHelper.isFilePatternExist(testResultsFilePattern)){
 
             try {
-                List<Map.Entry<String, ByteArrayInputStream>> artifacts = TestResultsHelper.extractArtifacts(artifactFiles,testResultsFilePattern);
+                List<Map.Entry<String, String>> artifacts = TestResultsHelper.extractArtifacts(artifactFiles,testResultsFilePattern);
                 if(artifacts!=null && !artifacts.isEmpty()){
                     TestResultsHelper.pushTestResultsKey(project,job);
                     return true;
@@ -154,8 +153,8 @@ public boolean createTestList(Project project, Job job, InputStream artifactFile
         return false;
     }
 
-    private void unmarshallAndAddToResults(List<TestRun> result, JAXBContext jaxbContext, ByteArrayInputStream artifact) throws JAXBException {
-        Object ots = jaxbContext.createUnmarshaller().unmarshal(artifact);
+    private void unmarshallAndAddToResults(List<TestRun> result, JAXBContext jaxbContext, String artifact) throws JAXBException {
+        Object ots = jaxbContext.createUnmarshaller().unmarshal(new StringReader(artifact));
         if (ots instanceof Testsuites) {
             ((Testsuites) ots).getTestsuite().forEach(ts -> ts.getTestcase().forEach(tc -> addTestCase(result, ts, tc)));
         } else if (ots instanceof Testsuite) {
@@ -167,7 +166,7 @@ private void addTestCase(List<TestRun> result, Testsuite ts, Testcase tc) {
         TestRunResult testResultStatus;
         if (tc.getSkipped() != null && tc.getSkipped().trim().length() > 0) {
             testResultStatus = TestRunResult.SKIPPED;
-        } else if (tc.getFailure().size() > 0) {
+        } else if (tc.getFailure().size() > 0 || tc.getError().size() > 0) {
             testResultStatus = TestRunResult.FAILED;
         } else {
             testResultStatus = TestRunResult.PASSED;