Merge pull request #270 from MarkUsProject/v1.10.2-rc

V1.10.2 rc

.gitignore

@@ -26,8 +26,7 @@ src/autotester/testers/*/specs/.installed
 src/autotester/testers/*/specs/install_settings.json
 
 # java
-src/autotester/testers/java/lib/.gradle
-src/autotester/testers/java/lib/build
+src/autotester/testers/java/lib/*.jar
 
 # racket
 src/autotester/testers/racket/**/compiled/

Changelog.md

@@ -1,6 +1,9 @@
 # CHANGELOG
 All notable changes to this project will be documented here.
 
+## [v1.10.2]
+- Updated java tester to support configurable classpaths and source files (#268)
+
 ## [v1.10.1]
 - Fixed bug where relevant test data was not included in the client while enqueuing tests (#265)
 
@@ -9,6 +12,7 @@ All notable changes to this project will be documented here.
 - Removed Tasty-Stats as a dependency for the haskell tester and added our own ingredient instead to collect stats (#259)
 - Updated/improved the interface between the autotester and clients that use it (#257)
 
+
 ## [1.9.0]
 - allow tests to write to existing subdirectories but not overwrite existing test script files (#237).
 - add ability to create a docker container for the autotester in development mode (#236).

src/autotester/testers/java/bin/install.sh

@@ -15,16 +15,9 @@ install_packages() {
     sudo DEBIAN_FRONTEND=${debian_frontend} apt-get ${apt_yes} "${apt_opts[@]}" install openjdk-8-jdk
 }
 
-compile_tester() {
-    echo "[JAVA-INSTALL] Compiling tester"
-    pushd "${JAVADIR}" > /dev/null
-    ./gradlew installDist --no-daemon
-    popd > /dev/null
-}
-
-update_specs() {
-    echo "[JAVA-INSTALL] Updating specs"
-    echo '{}' | jq ".path_to_tester_jars = \"${JAVADIR}/build/install/JavaTester/lib\"" > "${SPECSDIR}/install_settings.json"
+install_requirements() {
+    echo "[JAVA-INSTALL] Installing requirements"
+    wget https://repo1.maven.org/maven2/org/junit/platform/junit-platform-console-standalone/1.7.0/junit-platform-console-standalone-1.7.0.jar -O "${JAVADIR}/junit-platform-console-standalone.jar"
 }
 
 # script starts here
@@ -42,6 +35,5 @@ NON_INTERACTIVE=$1
 
 # main
 install_packages
-compile_tester
-update_specs
+install_requirements
 touch "${SPECSDIR}/.installed"

src/autotester/testers/java/bin/uninstall.sh

@@ -2,13 +2,7 @@
 
 remove_tester() {
     echo "[JAVA-UNINSTALL] Removing compiled tester"
-    rm -rf "${JAVADIR}/build"
-    rm -rf "${JAVADIR}/.gradle"
-}
-
-reset_specs() {
-    echo "[JAVA-UNINSTALL] Resetting specs"
-    rm -f "${SPECSDIR}/install_settings.json"
+    rm -r "${JAVADIR:?}/junit-platform-console-standalone.jar"
 }
 
 # script starts here
@@ -26,5 +20,5 @@ JAVADIR=$(readlink -f "${THISDIR}/../lib")
 # main
 remove_tester
 reset_specs
-echo "[JAVA-UNINSTALL] The following system packages have not been uninstalled: python3 openjdk-12-jdk jq. You may uninstall them if you wish."
+echo "[JAVA-UNINSTALL] The following system packages have not been uninstalled: python3 openjdk-8-jdk jq. You may uninstall them if you wish."
 rm -f "${SPECSDIR}/.installed"

src/autotester/testers/java/java_tester.py

@@ -1,60 +1,39 @@
-import enum
-import json
+import os
 import subprocess
-from typing import Dict, Optional, IO, Type
-
+import tempfile
+import xml.etree.ElementTree as eTree
+from glob import glob
+from typing import Type, List, Set
 from testers.specs import TestSpecs
 from testers.tester import Tester, Test, TestError
 
 
 class JavaTest(Test):
-    class JUnitStatus(enum.Enum):
-        SUCCESSFUL = 1
-        ABORTED = 2
-        FAILED = 3
-
-    ERRORS = {
-        "bad_javac": 'Java compilation error: "{}"',
-        "bad_java": 'Java runtime error: "{}"',
-    }
-
-    def __init__(self, tester: "JavaTester", result: Dict, feedback_open: Optional[IO] = None,) -> None:
-        """
-        Initialize a Java test created by tester.
-
-        The result was created after running some junit tests.
-        Test feedback will be written to feedback_open.
-        """
-        self.class_name, _sep, self.method_name = result["name"].partition(".")
-        self.description = result.get("description")
-        self.status = JavaTest.JUnitStatus[result["status"]]
-        self.message = result.get("message")
+    def __init__(self, tester, result, feedback_open=None):
+        self._test_name = result['name']
+        self.status = result['status']
+        self.message = result['message']
         super().__init__(tester, feedback_open)
 
     @property
-    def test_name(self) -> str:
-        """ The name of this test """
-        name = f"{self.class_name}.{self.method_name}"
-        if self.description:
-            name += f" ({self.description})"
-        return name
+    def test_name(self):
+        return self._test_name
 
     @Test.run_decorator
-    def run(self) -> str:
-        """
-        Return a json string containing all test result information.
-        """
-        if self.status == JavaTest.JUnitStatus.SUCCESSFUL:
-            return self.passed()
-        elif self.status == JavaTest.JUnitStatus.FAILED:
+    def run(self):
+        if self.status == "success":
+            return self.passed(message=self.message)
+        elif self.status == "failure":
             return self.failed(message=self.message)
         else:
             return self.error(message=self.message)
 
 
 class JavaTester(Tester):
 
-    JAVA_TESTER_CLASS = "edu.toronto.cs.teach.JavaTester"
+    JUNIT_TESTER_JAR = os.path.join(os.path.dirname(__file__), "lib", "junit-platform-console-standalone.jar")
+    JUNIT_JUPITER_RESULT = "TEST-junit-jupiter.xml"
+    JUNIT_VINTAGE_RESULT = "TEST-junit-vintage.xml"
 
     def __init__(self, specs: TestSpecs, test_class: Type[JavaTest] = JavaTest) -> None:
         """
@@ -63,17 +42,62 @@ def __init__(self, specs: TestSpecs, test_class: Type[JavaTest] = JavaTest) -> N
         This tester will create tests of type test_class.
         """
         super().__init__(specs, test_class)
-        self.java_classpath = f'.:{self.specs["install_data", "path_to_tester_jars"]}/*'
+        classpath = self.specs.get("test_data", "classpath", default='.') or '.'
+        self.java_classpath = ":".join(self._parse_file_paths(classpath))
+        self.out_dir = tempfile.TemporaryDirectory(dir=os.getcwd())
+        self.reports_dir = tempfile.TemporaryDirectory(dir=os.getcwd())
 
-    def compile(self) -> None:
+    @staticmethod
+    def _parse_file_paths(glob_string: str) -> List[str]:
+        """
+        Return the real (absolute) paths of all files described by the glob <glob_string>.
+        Only files that exist in the current directory (or its subdirectories) are returned.
+        """
+        curr_path = os.path.realpath('.')
+        return [x for p in glob_string.split(':') for x in glob(os.path.realpath(p)) if curr_path in x]
+
+    def _get_sources(self) -> Set:
+        """
+        Return all java source files for this test.
+        """
+        sources = self.specs.get("test_data", "sources_path", default='')
+        scripts = ':'.join(self.specs["test_data", "script_files"] + [sources])
+        return {path for path in self._parse_file_paths(scripts) if os.path.splitext(path)[1] == '.java'}
+
+    def _parse_junitxml(self):
+        """
+        Parse junit results and yield a hash containing result data for each testcase.
+        """
+        for xml_filename in [self.JUNIT_JUPITER_RESULT, self.JUNIT_VINTAGE_RESULT]:
+            tree = eTree.parse(os.path.join(self.reports_dir.name, xml_filename))
+            root = tree.getroot()
+            for testcase in root.iterfind('testcase'):
+                result = {}
+                classname = testcase.attrib['classname']
+                testname = testcase.attrib['name']
+                result['name'] = '{}.{}'.format(classname, testname)
+                result['time'] = float(testcase.attrib.get('time', 0))
+                failure = testcase.find('failure')
+                if failure is not None:
+                    result['status'] = 'failure'
+                    failure_type = failure.attrib.get('type', '')
+                    failure_message = failure.attrib.get('message', '')
+                    result['message'] = f'{failure_type}: {failure_message}'
+                else:
+                    result['status'] = 'success'
+                    result['message'] = ''
+                yield result
+
+    def compile(self) -> subprocess.CompletedProcess:
         """
         Compile the junit tests specified in the self.specs specifications.
         """
-        javac_command = ["javac", "-cp", self.java_classpath]
-        javac_command.extend(self.specs["test_data", "script_files"])
+        classpath = f"{self.java_classpath}:{self.JUNIT_TESTER_JAR}"
+        javac_command = ["javac", "-cp", classpath, "-d", self.out_dir.name]
+        javac_command.extend(self._get_sources())
         # student files imported by tests will be compiled on cascade
-        subprocess.run(
-            javac_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, check=True,
+        return subprocess.run(
+            javac_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, check=False,
         )
 
     def run_junit(self) -> subprocess.CompletedProcess:
@@ -82,13 +106,19 @@ def run_junit(self) -> subprocess.CompletedProcess:
         """
         java_command = [
             "java",
-            "-cp",
-            self.java_classpath,
-            JavaTester.JAVA_TESTER_CLASS,
+            "-jar",
+            self.JUNIT_TESTER_JAR,
+            f"-cp={self.java_classpath}:{self.out_dir.name}",
+            f"--reports-dir={self.reports_dir.name}"
         ]
-        java_command.extend(self.specs["test_data", "script_files"])
+        classes = [f"-c={os.path.splitext(os.path.basename(f))[0]}" for f in self.specs["test_data", "script_files"]]
+        java_command.extend(classes)
         java = subprocess.run(
-            java_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, check=True,
+            java_command,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            universal_newlines=True,
+            check=False
         )
         return java
 
@@ -99,20 +129,20 @@ def run(self) -> None:
         """
         # check that the submission compiles against the tests
         try:
-            self.compile()
+            compile_result = self.compile()
+            if compile_result.stderr:
+                raise TestError(compile_result.stderr)
         except subprocess.CalledProcessError as e:
-            msg = JavaTest.ERRORS["bad_javac"].format(e.stdout)
-            raise TestError(msg) from e
+            raise TestError(e)
         # run the tests with junit
         try:
             results = self.run_junit()
             if results.stderr:
                 raise TestError(results.stderr)
         except subprocess.CalledProcessError as e:
-            msg = JavaTest.ERRORS["bad_java"].format(e.stdout + e.stderr)
-            raise TestError(msg) from e
+            raise TestError(e)
         with self.open_feedback() as feedback_open:
-            for result in json.loads(results.stdout):
+            for result in self._parse_junitxml():
                 test = self.test_class(self, result, feedback_open)
                 result_json = test.run()
                 print(result_json, flush=True)