diff --git a/native-gradle-plugin/src/functionalTest/groovy/org/graalvm/buildtools/gradle/JavaApplicationWithTestsFunctionalTest.groovy b/native-gradle-plugin/src/functionalTest/groovy/org/graalvm/buildtools/gradle/JavaApplicationWithTestsFunctionalTest.groovy
index fc6ab60bf..733ff48bd 100644
--- a/native-gradle-plugin/src/functionalTest/groovy/org/graalvm/buildtools/gradle/JavaApplicationWithTestsFunctionalTest.groovy
+++ b/native-gradle-plugin/src/functionalTest/groovy/org/graalvm/buildtools/gradle/JavaApplicationWithTestsFunctionalTest.groovy
@@ -253,4 +253,38 @@ class JavaApplicationWithTestsFunctionalTest extends AbstractFunctionalTest {
         where:
         junitVersion = System.getProperty('versions.junit')
     }
+
+    @Issue("https://github.com/graalvm/native-build-tools/issues/218")
+    def "test list directory output is properly configured"() {
+        given:
+        withSample("java-application-with-tests")
+
+        buildFile << """
+            tasks.register("otherTest", Test) {
+                classpath = tasks.test.classpath
+                testClassesDirs = tasks.test.testClassesDirs
+            }
+        """
+
+        when:
+        run 'test'
+
+        then:
+        file("build/test-results/test/testlist").isDirectory()
+        !file("build/test-results/otherTest/testlist").isDirectory()
+
+        when:
+        run 'otherTest'
+
+        then:
+        !file("build/test-results/otherTest/testlist").isDirectory()
+
+        when:
+        run 'test'
+
+        then:
+        tasks {
+            upToDate(':test')
+        }
+    }
 }
diff --git a/native-gradle-plugin/src/main/java/org/graalvm/buildtools/gradle/NativeImagePlugin.java b/native-gradle-plugin/src/main/java/org/graalvm/buildtools/gradle/NativeImagePlugin.java
index 72ff61bd3..1dc8e2d17 100644
--- a/native-gradle-plugin/src/main/java/org/graalvm/buildtools/gradle/NativeImagePlugin.java
+++ b/native-gradle-plugin/src/main/java/org/graalvm/buildtools/gradle/NativeImagePlugin.java
@@ -83,10 +83,8 @@
 import org.gradle.api.provider.ListProperty;
 import org.gradle.api.provider.Property;
 import org.gradle.api.provider.Provider;
-import org.gradle.api.tasks.InputFiles;
 import org.gradle.api.tasks.JavaExec;
-import org.gradle.api.tasks.PathSensitive;
-import org.gradle.api.tasks.PathSensitivity;
+import org.gradle.api.tasks.OutputDirectory;
 import org.gradle.api.tasks.SourceSet;
 import org.gradle.api.tasks.SourceSetContainer;
 import org.gradle.api.tasks.TaskContainer;
@@ -96,8 +94,8 @@
 import org.gradle.api.tasks.testing.Test;
 import org.gradle.jvm.toolchain.JavaToolchainService;
 import org.gradle.language.base.plugins.LifecycleBasePlugin;
-import org.gradle.process.ExecOperations;
 import org.gradle.process.CommandLineArgumentProvider;
+import org.gradle.process.ExecOperations;
 import org.gradle.process.JavaForkOptions;
 import org.gradle.util.GFileUtils;
 
@@ -380,8 +378,9 @@ public void registerTestBinary(Project project,
         TaskProvider<Test> testTask = config.validate().getTestTask();
         testOptions.getAgent().getInstrumentedTask().set(testTask);
         testTask.configure(test -> {
-            testListDirectory.set(new File(testResultsDir, test.getName() + "/testlist"));
-            test.getOutputs().dir(testResultsDir);
+            File testList = new File(testResultsDir, test.getName() + "/testlist");
+            testListDirectory.set(testList);
+            test.getOutputs().dir(testList);
             // Set system property read by the UniqueIdTrackingListener.
             test.systemProperty(JUNIT_PLATFORM_LISTENERS_UID_TRACKING_ENABLED, true);
             TrackingDirectorySystemPropertyProvider directoryProvider = project.getObjects().newInstance(TrackingDirectorySystemPropertyProvider.class);
@@ -588,8 +587,7 @@ public void execute(Task task) {
     }
 
     public abstract static class TrackingDirectorySystemPropertyProvider implements CommandLineArgumentProvider {
-        @InputFiles
-        @PathSensitive(PathSensitivity.RELATIVE)
+        @OutputDirectory
         public abstract DirectoryProperty getDirectory();
 
         /**
diff --git a/native-gradle-plugin/src/testFixtures/groovy/org/graalvm/buildtools/gradle/fixtures/AbstractFunctionalTest.groovy b/native-gradle-plugin/src/testFixtures/groovy/org/graalvm/buildtools/gradle/fixtures/AbstractFunctionalTest.groovy
index 2df145387..6a4638603 100644
--- a/native-gradle-plugin/src/testFixtures/groovy/org/graalvm/buildtools/gradle/fixtures/AbstractFunctionalTest.groovy
+++ b/native-gradle-plugin/src/testFixtures/groovy/org/graalvm/buildtools/gradle/fixtures/AbstractFunctionalTest.groovy
@@ -231,6 +231,13 @@ abstract class AbstractFunctionalTest extends Specification {
             }
         }
 
+        void upToDate(String... tasks) {
+            tasks.each { task ->
+                contains(task)
+                assert result.task(task).outcome == TaskOutcome.UP_TO_DATE
+            }
+        }
+
         void contains(String... tasks) {
             tasks.each { task ->
                 assert result.task(task) != null: "Expected to find task $task in the graph but it was missing"