diff --git a/.gitignore b/.gitignore index 1a0e31af..6ccc47fa 100644 --- a/.gitignore +++ b/.gitignore @@ -26,8 +26,7 @@ src/autotester/testers/*/specs/.installed src/autotester/testers/*/specs/install_settings.json # java -src/autotester/testers/java/lib/.gradle -src/autotester/testers/java/lib/build +src/autotester/testers/java/lib/*.jar # racket src/autotester/testers/racket/**/compiled/ diff --git a/Changelog.md b/Changelog.md index 3516d291..3dedf658 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,6 +1,9 @@ # CHANGELOG All notable changes to this project will be documented here. +## [v1.10.2] +- Updated java tester to support configurable classpaths and source files (#268) + ## [v1.10.1] - Fixed bug where relevant test data was not included in the client while enqueuing tests (#265) @@ -9,6 +12,7 @@ All notable changes to this project will be documented here. - Removed Tasty-Stats as a dependency for the haskell tester and added our own ingredient instead to collect stats (#259) - Updated/improved the interface between the autotester and clients that use it (#257) + ## [1.9.0] - allow tests to write to existing subdirectories but not overwrite existing test script files (#237). - add ability to create a docker container for the autotester in development mode (#236). diff --git a/src/autotester/testers/java/bin/install.sh b/src/autotester/testers/java/bin/install.sh index a9ab6e19..42be0e60 100755 --- a/src/autotester/testers/java/bin/install.sh +++ b/src/autotester/testers/java/bin/install.sh @@ -15,16 +15,9 @@ install_packages() { sudo DEBIAN_FRONTEND=${debian_frontend} apt-get ${apt_yes} "${apt_opts[@]}" install openjdk-8-jdk } -compile_tester() { - echo "[JAVA-INSTALL] Compiling tester" - pushd "${JAVADIR}" > /dev/null - ./gradlew installDist --no-daemon - popd > /dev/null -} - -update_specs() { - echo "[JAVA-INSTALL] Updating specs" - echo '{}' | jq ".path_to_tester_jars = \"${JAVADIR}/build/install/JavaTester/lib\"" > "${SPECSDIR}/install_settings.json" +install_requirements() { + echo "[JAVA-INSTALL] Installing requirements" + wget https://repo1.maven.org/maven2/org/junit/platform/junit-platform-console-standalone/1.7.0/junit-platform-console-standalone-1.7.0.jar -O "${JAVADIR}/junit-platform-console-standalone.jar" } # script starts here @@ -42,6 +35,5 @@ NON_INTERACTIVE=$1 # main install_packages -compile_tester -update_specs +install_requirements touch "${SPECSDIR}/.installed" diff --git a/src/autotester/testers/java/bin/uninstall.sh b/src/autotester/testers/java/bin/uninstall.sh index 9fad619d..8d2954d8 100755 --- a/src/autotester/testers/java/bin/uninstall.sh +++ b/src/autotester/testers/java/bin/uninstall.sh @@ -2,13 +2,7 @@ remove_tester() { echo "[JAVA-UNINSTALL] Removing compiled tester" - rm -rf "${JAVADIR}/build" - rm -rf "${JAVADIR}/.gradle" -} - -reset_specs() { - echo "[JAVA-UNINSTALL] Resetting specs" - rm -f "${SPECSDIR}/install_settings.json" + rm -r "${JAVADIR:?}/junit-platform-console-standalone.jar" } # script starts here @@ -26,5 +20,5 @@ JAVADIR=$(readlink -f "${THISDIR}/../lib") # main remove_tester reset_specs -echo "[JAVA-UNINSTALL] The following system packages have not been uninstalled: python3 openjdk-12-jdk jq. You may uninstall them if you wish." +echo "[JAVA-UNINSTALL] The following system packages have not been uninstalled: python3 openjdk-8-jdk jq. You may uninstall them if you wish." rm -f "${SPECSDIR}/.installed" diff --git a/src/autotester/testers/java/java_tester.py b/src/autotester/testers/java/java_tester.py index 25b07e40..d82aa34a 100644 --- a/src/autotester/testers/java/java_tester.py +++ b/src/autotester/testers/java/java_tester.py @@ -1,52 +1,29 @@ -import enum -import json +import os import subprocess -from typing import Dict, Optional, IO, Type - +import tempfile +import xml.etree.ElementTree as eTree +from glob import glob +from typing import Type, List, Set from testers.specs import TestSpecs from testers.tester import Tester, Test, TestError class JavaTest(Test): - class JUnitStatus(enum.Enum): - SUCCESSFUL = 1 - ABORTED = 2 - FAILED = 3 - - ERRORS = { - "bad_javac": 'Java compilation error: "{}"', - "bad_java": 'Java runtime error: "{}"', - } - - def __init__(self, tester: "JavaTester", result: Dict, feedback_open: Optional[IO] = None,) -> None: - """ - Initialize a Java test created by tester. - - The result was created after running some junit tests. - Test feedback will be written to feedback_open. - """ - self.class_name, _sep, self.method_name = result["name"].partition(".") - self.description = result.get("description") - self.status = JavaTest.JUnitStatus[result["status"]] - self.message = result.get("message") + def __init__(self, tester, result, feedback_open=None): + self._test_name = result['name'] + self.status = result['status'] + self.message = result['message'] super().__init__(tester, feedback_open) @property - def test_name(self) -> str: - """ The name of this test """ - name = f"{self.class_name}.{self.method_name}" - if self.description: - name += f" ({self.description})" - return name + def test_name(self): + return self._test_name @Test.run_decorator - def run(self) -> str: - """ - Return a json string containing all test result information. - """ - if self.status == JavaTest.JUnitStatus.SUCCESSFUL: - return self.passed() - elif self.status == JavaTest.JUnitStatus.FAILED: + def run(self): + if self.status == "success": + return self.passed(message=self.message) + elif self.status == "failure": return self.failed(message=self.message) else: return self.error(message=self.message) @@ -54,7 +31,9 @@ def run(self) -> str: class JavaTester(Tester): - JAVA_TESTER_CLASS = "edu.toronto.cs.teach.JavaTester" + JUNIT_TESTER_JAR = os.path.join(os.path.dirname(__file__), "lib", "junit-platform-console-standalone.jar") + JUNIT_JUPITER_RESULT = "TEST-junit-jupiter.xml" + JUNIT_VINTAGE_RESULT = "TEST-junit-vintage.xml" def __init__(self, specs: TestSpecs, test_class: Type[JavaTest] = JavaTest) -> None: """ @@ -63,17 +42,62 @@ def __init__(self, specs: TestSpecs, test_class: Type[JavaTest] = JavaTest) -> N This tester will create tests of type test_class. """ super().__init__(specs, test_class) - self.java_classpath = f'.:{self.specs["install_data", "path_to_tester_jars"]}/*' + classpath = self.specs.get("test_data", "classpath", default='.') or '.' + self.java_classpath = ":".join(self._parse_file_paths(classpath)) + self.out_dir = tempfile.TemporaryDirectory(dir=os.getcwd()) + self.reports_dir = tempfile.TemporaryDirectory(dir=os.getcwd()) - def compile(self) -> None: + @staticmethod + def _parse_file_paths(glob_string: str) -> List[str]: + """ + Return the real (absolute) paths of all files described by the glob . + Only files that exist in the current directory (or its subdirectories) are returned. + """ + curr_path = os.path.realpath('.') + return [x for p in glob_string.split(':') for x in glob(os.path.realpath(p)) if curr_path in x] + + def _get_sources(self) -> Set: + """ + Return all java source files for this test. + """ + sources = self.specs.get("test_data", "sources_path", default='') + scripts = ':'.join(self.specs["test_data", "script_files"] + [sources]) + return {path for path in self._parse_file_paths(scripts) if os.path.splitext(path)[1] == '.java'} + + def _parse_junitxml(self): + """ + Parse junit results and yield a hash containing result data for each testcase. + """ + for xml_filename in [self.JUNIT_JUPITER_RESULT, self.JUNIT_VINTAGE_RESULT]: + tree = eTree.parse(os.path.join(self.reports_dir.name, xml_filename)) + root = tree.getroot() + for testcase in root.iterfind('testcase'): + result = {} + classname = testcase.attrib['classname'] + testname = testcase.attrib['name'] + result['name'] = '{}.{}'.format(classname, testname) + result['time'] = float(testcase.attrib.get('time', 0)) + failure = testcase.find('failure') + if failure is not None: + result['status'] = 'failure' + failure_type = failure.attrib.get('type', '') + failure_message = failure.attrib.get('message', '') + result['message'] = f'{failure_type}: {failure_message}' + else: + result['status'] = 'success' + result['message'] = '' + yield result + + def compile(self) -> subprocess.CompletedProcess: """ Compile the junit tests specified in the self.specs specifications. """ - javac_command = ["javac", "-cp", self.java_classpath] - javac_command.extend(self.specs["test_data", "script_files"]) + classpath = f"{self.java_classpath}:{self.JUNIT_TESTER_JAR}" + javac_command = ["javac", "-cp", classpath, "-d", self.out_dir.name] + javac_command.extend(self._get_sources()) # student files imported by tests will be compiled on cascade - subprocess.run( - javac_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, check=True, + return subprocess.run( + javac_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, check=False, ) def run_junit(self) -> subprocess.CompletedProcess: @@ -82,13 +106,19 @@ def run_junit(self) -> subprocess.CompletedProcess: """ java_command = [ "java", - "-cp", - self.java_classpath, - JavaTester.JAVA_TESTER_CLASS, + "-jar", + self.JUNIT_TESTER_JAR, + f"-cp={self.java_classpath}:{self.out_dir.name}", + f"--reports-dir={self.reports_dir.name}" ] - java_command.extend(self.specs["test_data", "script_files"]) + classes = [f"-c={os.path.splitext(os.path.basename(f))[0]}" for f in self.specs["test_data", "script_files"]] + java_command.extend(classes) java = subprocess.run( - java_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, check=True, + java_command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + check=False ) return java @@ -99,20 +129,20 @@ def run(self) -> None: """ # check that the submission compiles against the tests try: - self.compile() + compile_result = self.compile() + if compile_result.stderr: + raise TestError(compile_result.stderr) except subprocess.CalledProcessError as e: - msg = JavaTest.ERRORS["bad_javac"].format(e.stdout) - raise TestError(msg) from e + raise TestError(e) # run the tests with junit try: results = self.run_junit() if results.stderr: raise TestError(results.stderr) except subprocess.CalledProcessError as e: - msg = JavaTest.ERRORS["bad_java"].format(e.stdout + e.stderr) - raise TestError(msg) from e + raise TestError(e) with self.open_feedback() as feedback_open: - for result in json.loads(results.stdout): + for result in self._parse_junitxml(): test = self.test_class(self, result, feedback_open) result_json = test.run() print(result_json, flush=True) diff --git a/src/autotester/testers/java/lib/.gitkeep b/src/autotester/testers/java/lib/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/src/autotester/testers/java/lib/build.gradle b/src/autotester/testers/java/lib/build.gradle deleted file mode 100644 index a2331592..00000000 --- a/src/autotester/testers/java/lib/build.gradle +++ /dev/null @@ -1,20 +0,0 @@ -apply plugin: 'java' -apply plugin: 'application' - -group = 'edu.toronto.cs.teach' -version = '1.7.1' -sourceCompatibility = 1.8 -mainClassName = 'edu.toronto.cs.teach.JavaTester' - -repositories { - mavenCentral() -} - -dependencies { - implementation group: 'org.jetbrains', name: 'annotations', version: '17.0.0' - implementation group: 'org.junit.platform', name: 'junit-platform-engine', version: '1.4.1' - implementation group: 'org.junit.platform', name: 'junit-platform-launcher', version: '1.4.1' - implementation group: 'org.junit.jupiter', name: 'junit-jupiter-api', version: '5.4.1' - implementation group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: '5.4.1' - implementation group: 'com.google.code.gson', name: 'gson', version: '2.8.5' -} diff --git a/src/autotester/testers/java/lib/gradle/wrapper/gradle-wrapper.jar b/src/autotester/testers/java/lib/gradle/wrapper/gradle-wrapper.jar deleted file mode 100644 index 5c2d1cf0..00000000 Binary files a/src/autotester/testers/java/lib/gradle/wrapper/gradle-wrapper.jar and /dev/null differ diff --git a/src/autotester/testers/java/lib/gradle/wrapper/gradle-wrapper.properties b/src/autotester/testers/java/lib/gradle/wrapper/gradle-wrapper.properties deleted file mode 100644 index 51fb1c46..00000000 --- a/src/autotester/testers/java/lib/gradle/wrapper/gradle-wrapper.properties +++ /dev/null @@ -1,5 +0,0 @@ -distributionBase=GRADLE_USER_HOME -distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-5.3.1-all.zip -zipStoreBase=GRADLE_USER_HOME -zipStorePath=wrapper/dists diff --git a/src/autotester/testers/java/lib/gradlew b/src/autotester/testers/java/lib/gradlew deleted file mode 100755 index 916b9742..00000000 --- a/src/autotester/testers/java/lib/gradlew +++ /dev/null @@ -1,188 +0,0 @@ -#!/usr/bin/env sh - -# -# Copyright 2015 the original author or authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -############################################################################## -## -## Gradle start up script for UN*X -## -############################################################################## - -# Attempt to set APP_HOME -# Resolve links: $0 may be a link -PRG="$0" -# Need this for relative symlinks. -while [ -h "$PRG" ] ; do - ls=`ls -ld "$PRG"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG=`dirname "$PRG"`"/$link" - fi -done -SAVED="`pwd`" -cd "`dirname \"$PRG\"`/" >/dev/null -APP_HOME="`pwd -P`" -cd "$SAVED" >/dev/null - -APP_NAME="Gradle" -APP_BASE_NAME=`basename "$0"` - -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS='' - -# Use the maximum available, or set MAX_FD != -1 to use that value. -MAX_FD="maximum" - -warn () { - echo "$*" -} - -die () { - echo - echo "$*" - echo - exit 1 -} - -# OS specific support (must be 'true' or 'false'). -cygwin=false -msys=false -darwin=false -nonstop=false -case "`uname`" in - CYGWIN* ) - cygwin=true - ;; - Darwin* ) - darwin=true - ;; - MINGW* ) - msys=true - ;; - NONSTOP* ) - nonstop=true - ;; -esac - -CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar - -# Determine the Java command to use to start the JVM. -if [ -n "$JAVA_HOME" ] ; then - if [ -x "$JAVA_HOME/jre/sh/java" ] ; then - # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" - else - JAVACMD="$JAVA_HOME/bin/java" - fi - if [ ! -x "$JAVACMD" ] ; then - die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME - -Please set the JAVA_HOME variable in your environment to match the -location of your Java installation." - fi -else - JAVACMD="java" - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. - -Please set the JAVA_HOME variable in your environment to match the -location of your Java installation." -fi - -# Increase the maximum file descriptors if we can. -if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then - MAX_FD_LIMIT=`ulimit -H -n` - if [ $? -eq 0 ] ; then - if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then - MAX_FD="$MAX_FD_LIMIT" - fi - ulimit -n $MAX_FD - if [ $? -ne 0 ] ; then - warn "Could not set maximum file descriptor limit: $MAX_FD" - fi - else - warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" - fi -fi - -# For Darwin, add options to specify how the application appears in the dock -if $darwin; then - GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" -fi - -# For Cygwin, switch paths to Windows format before running java -if $cygwin ; then - APP_HOME=`cygpath --path --mixed "$APP_HOME"` - CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` - JAVACMD=`cygpath --unix "$JAVACMD"` - - # We build the pattern for arguments to be converted via cygpath - ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` - SEP="" - for dir in $ROOTDIRSRAW ; do - ROOTDIRS="$ROOTDIRS$SEP$dir" - SEP="|" - done - OURCYGPATTERN="(^($ROOTDIRS))" - # Add a user-defined pattern to the cygpath arguments - if [ "$GRADLE_CYGPATTERN" != "" ] ; then - OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" - fi - # Now convert the arguments - kludge to limit ourselves to /bin/sh - i=0 - for arg in "$@" ; do - CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` - CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option - - if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition - eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` - else - eval `echo args$i`="\"$arg\"" - fi - i=$((i+1)) - done - case $i in - (0) set -- ;; - (1) set -- "$args0" ;; - (2) set -- "$args0" "$args1" ;; - (3) set -- "$args0" "$args1" "$args2" ;; - (4) set -- "$args0" "$args1" "$args2" "$args3" ;; - (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; - (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; - (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; - (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; - (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; - esac -fi - -# Escape application args -save () { - for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done - echo " " -} -APP_ARGS=$(save "$@") - -# Collect all arguments for the java command, following the shell quoting and substitution rules -eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" - -# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong -if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then - cd "$(dirname "$0")" -fi - -exec "$JAVACMD" "$@" diff --git a/src/autotester/testers/java/lib/gradlew.bat b/src/autotester/testers/java/lib/gradlew.bat deleted file mode 100644 index 15e1ee37..00000000 --- a/src/autotester/testers/java/lib/gradlew.bat +++ /dev/null @@ -1,100 +0,0 @@ -@rem -@rem Copyright 2015 the original author or authors. -@rem -@rem Licensed under the Apache License, Version 2.0 (the "License"); -@rem you may not use this file except in compliance with the License. -@rem You may obtain a copy of the License at -@rem -@rem http://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, software -@rem distributed under the License is distributed on an "AS IS" BASIS, -@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -@rem See the License for the specific language governing permissions and -@rem limitations under the License. -@rem - -@if "%DEBUG%" == "" @echo off -@rem ########################################################################## -@rem -@rem Gradle startup script for Windows -@rem -@rem ########################################################################## - -@rem Set local scope for the variables with windows NT shell -if "%OS%"=="Windows_NT" setlocal - -set DIRNAME=%~dp0 -if "%DIRNAME%" == "" set DIRNAME=. -set APP_BASE_NAME=%~n0 -set APP_HOME=%DIRNAME% - -@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" - -@rem Find java.exe -if defined JAVA_HOME goto findJavaFromJavaHome - -set JAVA_EXE=java.exe -%JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto init - -echo. -echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. - -goto fail - -:findJavaFromJavaHome -set JAVA_HOME=%JAVA_HOME:"=% -set JAVA_EXE=%JAVA_HOME%/bin/java.exe - -if exist "%JAVA_EXE%" goto init - -echo. -echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. - -goto fail - -:init -@rem Get command-line arguments, handling Windows variants - -if not "%OS%" == "Windows_NT" goto win9xME_args - -:win9xME_args -@rem Slurp the command line arguments. -set CMD_LINE_ARGS= -set _SKIP=2 - -:win9xME_args_slurp -if "x%~1" == "x" goto execute - -set CMD_LINE_ARGS=%* - -:execute -@rem Setup the command line - -set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar - -@rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% - -:end -@rem End local scope for the variables with windows NT shell -if "%ERRORLEVEL%"=="0" goto mainEnd - -:fail -rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of -rem the _cmd.exe /c_ return code! -if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 -exit /b 1 - -:mainEnd -if "%OS%"=="Windows_NT" endlocal - -:omega diff --git a/src/autotester/testers/java/lib/settings.gradle b/src/autotester/testers/java/lib/settings.gradle deleted file mode 100644 index 71c5f8e1..00000000 --- a/src/autotester/testers/java/lib/settings.gradle +++ /dev/null @@ -1 +0,0 @@ -rootProject.name = 'JavaTester' diff --git a/src/autotester/testers/java/lib/src/main/java/edu/toronto/cs/teach/JavaTester.java b/src/autotester/testers/java/lib/src/main/java/edu/toronto/cs/teach/JavaTester.java deleted file mode 100644 index de1983a4..00000000 --- a/src/autotester/testers/java/lib/src/main/java/edu/toronto/cs/teach/JavaTester.java +++ /dev/null @@ -1,100 +0,0 @@ -package edu.toronto.cs.teach; - -import com.google.gson.Gson; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.junit.platform.engine.TestExecutionResult; -import org.junit.platform.engine.TestSource; -import org.junit.platform.engine.discovery.ClassNameFilter; -import org.junit.platform.engine.discovery.DiscoverySelectors; -import org.junit.platform.engine.support.descriptor.MethodSource; -import org.junit.platform.launcher.Launcher; -import org.junit.platform.launcher.LauncherDiscoveryRequest; -import org.junit.platform.launcher.TestExecutionListener; -import org.junit.platform.launcher.TestIdentifier; -import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder; -import org.junit.platform.launcher.core.LauncherFactory; - -import java.io.IOException; -import java.io.OutputStream; -import java.io.PrintStream; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.*; - -public class JavaTester { - - private class TestResult { - private @NotNull String name; - private @NotNull TestExecutionResult.Status status; - private @Nullable String description; - private @Nullable String message; - } - - private @NotNull String[] testClasses; - private @NotNull List results; - - public JavaTester(@NotNull String[] testFiles) { - - this.testClasses = new String[testFiles.length]; - for (int i = 0; i < testFiles.length; i++) { - testClasses[i] = testFiles[i].split("\\.")[0]; - } - this.results = new ArrayList<>(); - } - - private void run() { - - // redirect stdout and stderr - PrintStream outOrig = System.out, errOrig = System.err; - System.setOut(new PrintStream(new OutputStream() { - public void write(int b) throws IOException {} - })); - System.setErr(new PrintStream(new OutputStream() { - public void write(int b) throws IOException {} - })); - // run tests - Set classpath = new HashSet<>(); - classpath.add(Paths.get(".")); - LauncherDiscoveryRequest request = LauncherDiscoveryRequestBuilder.request() - .selectors(DiscoverySelectors.selectClasspathRoots(classpath)) - .filters(ClassNameFilter.includeClassNamePatterns(this.testClasses)) - .build(); - Launcher launcher = LauncherFactory.create(); - launcher.registerTestExecutionListeners(new TestExecutionListener() { - @Override - public void executionFinished(TestIdentifier testIdentifier, TestExecutionResult testExecutionResult) { - // record a single test result - if (testIdentifier.isContainer()) { - return; - } - TestSource source = testIdentifier.getSource().orElse(null); - if (source == null || !(source instanceof MethodSource)) { - return; - } - TestResult result = new TestResult(); - String className = ((MethodSource) source).getClassName(); - String methodName = ((MethodSource) source).getMethodName(); - result.name = className + "." + methodName; - if (!testIdentifier.getDisplayName().equals(methodName + "()")) { // @DisplayName annotation - result.description = testIdentifier.getDisplayName(); - } - result.status = testExecutionResult.getStatus(); - testExecutionResult.getThrowable().ifPresent(t -> result.message = t.getMessage()); - results.add(result); - } - }); - launcher.execute(request); - // restore stdout and stderr, then print results - System.setOut(outOrig); - System.setErr(errOrig); - System.out.print(new Gson().toJson(this.results)); - } - - public static void main(String[] args) { - - JavaTester tester = new JavaTester(args); - tester.run(); - } - -} diff --git a/src/autotester/testers/java/specs/settings_schema.json b/src/autotester/testers/java/specs/settings_schema.json index a823c0b9..2540af86 100644 --- a/src/autotester/testers/java/specs/settings_schema.json +++ b/src/autotester/testers/java/specs/settings_schema.json @@ -18,6 +18,14 @@ "timeout" ], "properties": { + "classpath": { + "title": "Java Class Path", + "type": "string" + }, + "sources_path": { + "title": "Java Sources (glob)", + "type": "string" + }, "script_files": { "title": "Test files", "type": "array", diff --git a/src/autotester/testers/java/tests/script_files/TestJunit4.java b/src/autotester/testers/java/tests/script_files/TestJunit4.java new file mode 100644 index 00000000..7e1c25c5 --- /dev/null +++ b/src/autotester/testers/java/tests/script_files/TestJunit4.java @@ -0,0 +1,22 @@ +import org.junit.DisplayName; +import org.junit.Test; + +import static org.junit.Assertions.assertTrue; + +public class Test1 { + + Submission submission = new Submission(); + + @Test + @DisplayName("This test should pass") + public void testPasses() { + assertTrue(submission.returnTrue()); + } + + @Test + @DisplayName("This test should fail") + public void testFails() { + assertTrue(submission.returnFalse()); + } + +} diff --git a/src/autotester/testers/java/tests/specs.json b/src/autotester/testers/java/tests/specs.json index 0a4c39cb..029fa8c3 100644 --- a/src/autotester/testers/java/tests/specs.json +++ b/src/autotester/testers/java/tests/specs.json @@ -4,14 +4,22 @@ "tester_type": "java", "test_data": [ { - "script_files": ["Test1.java"], + "script_files": ["script_files/Test1.java"], "category": ["admin"], - "timeout": 30 + "timeout": 30, + "sources_path": "./*" }, { - "script_files": ["Test2.java"], + "script_files": ["script_files/Test2.java"], "category": ["admin"], - "timeout": 30 + "timeout": 30, + "sources_path": "student_files/*" + }, + { + "script_files": ["script_files/TestJunit4.java"], + "category": ["admin"], + "timeout": 30, + "sources_path": "student_files/*" } ] }