diff --git a/compiler-plugin/src/main/kotlin/com/google/devtools/ksp/Incremental.kt b/compiler-plugin/src/main/kotlin/com/google/devtools/ksp/Incremental.kt
index 2de08ff2e4..90a25630f3 100644
--- a/compiler-plugin/src/main/kotlin/com/google/devtools/ksp/Incremental.kt
+++ b/compiler-plugin/src/main/kotlin/com/google/devtools/ksp/Incremental.kt
@@ -231,10 +231,11 @@ class IncrementalContext(
     // Ugly, but better than copying the private logics out of stdlib.
     // TODO: get rid of `relativeTo` if possible.
     private fun String.toRelativeFile(): File =
-        if (this.startsWith(buildDir.path))
+        try {
             File(this).relativeTo(buildDir)
-        else
+        } catch (e: Exception) {
             File(this).relativeTo(baseDir)
+        }
 
     private val KSFile.relativeFile
         get() = filePath.toRelativeFile()
diff --git a/integration-tests/src/test/kotlin/com/google/devtools/ksp/test/PlaygroundIT.kt b/integration-tests/src/test/kotlin/com/google/devtools/ksp/test/PlaygroundIT.kt
index 2e16375603..73640fa1e5 100644
--- a/integration-tests/src/test/kotlin/com/google/devtools/ksp/test/PlaygroundIT.kt
+++ b/integration-tests/src/test/kotlin/com/google/devtools/ksp/test/PlaygroundIT.kt
@@ -65,6 +65,28 @@ class PlaygroundIT {
         project.restore("workload/build.gradle.kts")
     }
 
+    @Test
+    fun testArbitraryBuildDir() {
+        Assume.assumeTrue(System.getProperty("os.name").startsWith("Windows", ignoreCase = true))
+        val gradleRunner = GradleRunner.create().withProjectDir(project.root)
+
+        File(project.root, "workload/build.gradle.kts")
+            .appendText("project.buildDir = File(\"D:/build/\")")
+        val result = gradleRunner.withArguments("build").build()
+
+        Assert.assertEquals(TaskOutcome.SUCCESS, result.task(":workload:build")?.outcome)
+
+        val artifact = File("D:/build/libs/workload-1.0-SNAPSHOT.jar")
+        Assert.assertTrue(artifact.exists())
+
+        JarFile(artifact).use { jarFile ->
+            Assert.assertTrue(jarFile.getEntry("TestProcessor.log").size > 0)
+            Assert.assertTrue(jarFile.getEntry("hello/HELLO.class").size > 0)
+            Assert.assertTrue(jarFile.getEntry("g/G.class").size > 0)
+            Assert.assertTrue(jarFile.getEntry("com/example/AClassBuilder.class").size > 0)
+        }
+    }
+
     @Test
     fun testAllowSourcesFromOtherPlugins() {
         fun checkGBuilder() {