Backport generation-compilation ordering fixes from #13282.

Correct whitespace difference in opennlp-tools-1.9.4.jar.sha1.

gradle/generation/antlr.gradle

@@ -82,7 +82,10 @@ configure(project(":lucene:expressions")) {
   }
 
   def generateAntlr = wrapWithPersistentChecksums(
-      generateAntlrInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])
+      generateAntlrInternal, [
+          andThenTasks: ["spotlessJava", "spotlessJavaApply"],
+          mustRunBefore: [ "compileJava" ]
+  ])
 
   regenerate.dependsOn generateAntlr
 }

gradle/generation/forUtil.gradle

@@ -39,7 +39,10 @@ configure(project(":lucene:core")) {
     }
   }
 
-  regenerate.dependsOn wrapWithPersistentChecksums(generateForUtilInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])
+  regenerate.dependsOn wrapWithPersistentChecksums(generateForUtilInternal, [
+          andThenTasks: ["spotlessJava", "spotlessJavaApply"],
+          mustRunBefore: [ "compileJava" ]
+  ])
 }
 
 configure(project(":lucene:backward-codecs")) {
@@ -64,7 +67,10 @@ configure(project(":lucene:backward-codecs")) {
     }
   }
 
-  regenerate.dependsOn wrapWithPersistentChecksums(generateForUtil84Internal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])
+  regenerate.dependsOn wrapWithPersistentChecksums(generateForUtil84Internal, [
+          andThenTasks: ["spotlessJava", "spotlessJavaApply"],
+          mustRunBefore: [ "compileJava" ]
+  ])
 
   task generateForUtil90Internal() {
     description "Regenerate gen_ForUtil.py"
@@ -86,6 +92,9 @@ configure(project(":lucene:backward-codecs")) {
     }
   }
 
-  regenerate.dependsOn wrapWithPersistentChecksums(generateForUtil90Internal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])
+  regenerate.dependsOn wrapWithPersistentChecksums(generateForUtil90Internal, [
+          andThenTasks: ["spotlessJava", "spotlessJavaApply"],
+          mustRunBefore: [ "compileJava" ]
+  ])
 }
 

gradle/generation/icu.gradle

@@ -159,8 +159,13 @@ configure(project(":lucene:analysis:icu")) {
     }
   }
 
-  regenerate.dependsOn wrapWithPersistentChecksums(genUtr30DataFilesInternal, [ ignoreWithSource: icuCompileTask ])
-  regenerate.dependsOn wrapWithPersistentChecksums(genRbbiInternal)
+  regenerate.dependsOn wrapWithPersistentChecksums(genUtr30DataFilesInternal, [
+    ignoreWithSource: icuCompileTask,
+    mustRunBefore: [ "compileJava" ]
+  ])
+  regenerate.dependsOn wrapWithPersistentChecksums(genRbbiInternal, [
+    mustRunBefore: [ "compileJava" ]
+  ])
 
   task compileIcuWindows() {
     doFirst {
@@ -281,5 +286,8 @@ configure(project(":lucene:analysis:common")) {
     }
   }
 
-  regenerate.dependsOn wrapWithPersistentChecksums(generateUnicodePropsInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])
+  regenerate.dependsOn wrapWithPersistentChecksums(generateUnicodePropsInternal, [
+          andThenTasks: ["spotlessJava", "spotlessJavaApply"],
+          mustRunBefore: [ "compileJava" ]
+  ])
 }

gradle/generation/javacc.gradle

@@ -232,9 +232,14 @@ configure(project(":lucene:queryparser")) {
     description "Regenerate query parsers (javacc syntax definitions)."
     group "generation"
 
-    dependsOn wrapWithPersistentChecksums(javaccParserClassicInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ]),
-        wrapWithPersistentChecksums(javaccParserSurroundInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ]),
-        wrapWithPersistentChecksums(javaccParserFlexibleInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])
+    def extraConfig = [
+        andThenTasks: ["spotlessJava", "spotlessJavaApply"],
+        mustRunBefore: [ "compileJava" ]
+    ]
+
+    dependsOn wrapWithPersistentChecksums(javaccParserClassicInternal, extraConfig),
+        wrapWithPersistentChecksums(javaccParserSurroundInternal, extraConfig),
+        wrapWithPersistentChecksums(javaccParserFlexibleInternal, extraConfig)
   }
 
   regenerate.dependsOn javacc

gradle/generation/jflex.gradle

@@ -51,7 +51,10 @@ configure(project(":lucene:core")) {
     }
   }
 
-  def generateStandardTokenizer = wrapWithPersistentChecksums(generateStandardTokenizerInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])
+  def generateStandardTokenizer = wrapWithPersistentChecksums(generateStandardTokenizerInternal, [
+          andThenTasks: ["spotlessJava", "spotlessJavaApply"],
+          mustRunBefore: [ "compileJava" ]
+  ])
 
   regenerate.dependsOn generateStandardTokenizer
 }
@@ -149,7 +152,11 @@ configure(project(":lucene:analysis:common")) {
     }
   }
 
-  def generateUAX29URLEmailTokenizer = wrapWithPersistentChecksums(generateUAX29URLEmailTokenizerInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])
+  def generateUAX29URLEmailTokenizer = wrapWithPersistentChecksums(generateUAX29URLEmailTokenizerInternal, [
+          andThenTasks: ["spotlessJava", "spotlessJavaApply"],
+          mustRunBefore: [ "compileJava" ]
+  ])
+
   // UAX29URLEmailTokenizerImpl.jflex includes: ASCIITLD.jflex
   // so we make sure it is up to date. Also, make sure the ordering of internal
   // tasks is enforced (just scheduling top-level wrappers doesn't mean their subtask graphs cannot be reordered).
@@ -197,7 +204,12 @@ configure(project(":lucene:analysis:common")) {
     skeleton = skeletonDefault
   }
 
-  def generateHTMLStripCharFilter = wrapWithPersistentChecksums(generateHTMLStripCharFilterInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])
+  def extraConfig = [
+      andThenTasks: ["spotlessJava", "spotlessJavaApply"],
+      mustRunBefore: [ "compileJava" ]
+  ]
+
+  def generateHTMLStripCharFilter = wrapWithPersistentChecksums(generateHTMLStripCharFilterInternal, extraConfig)
   // HTMLStripCharFilter.jflex includes HTMLCharacterEntities.jflex so we make sure it's up to date.
   configure(generateHTMLStripCharFilter) {
     dependsOn "generateHTMLCharacterEntities"
@@ -206,12 +218,12 @@ configure(project(":lucene:analysis:common")) {
     dependsOn "generateHTMLCharacterEntitiesInternal"
   }
 
-  regenerate.dependsOn wrapWithPersistentChecksums(generateWikipediaTokenizerInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ]),
-      wrapWithPersistentChecksums(generateClassicTokenizerInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ]),
+  regenerate.dependsOn wrapWithPersistentChecksums(generateWikipediaTokenizerInternal, extraConfig),
+      wrapWithPersistentChecksums(generateClassicTokenizerInternal, extraConfig),
       generateUAX29URLEmailTokenizer,
-      wrapWithPersistentChecksums(generateHTMLCharacterEntitiesInternal),
+      wrapWithPersistentChecksums(generateHTMLCharacterEntitiesInternal, extraConfig),
       generateHTMLStripCharFilter,
-      wrapWithPersistentChecksums(generateTldsInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])
+      wrapWithPersistentChecksums(generateTldsInternal, extraConfig)
 }
 
 class JFlexTask extends DefaultTask {

gradle/generation/moman.gradle

@@ -105,8 +105,13 @@ configure(project(":lucene:core")) {
     description "Regenerate Moman-based sources."
     group "generation"
 
-    dependsOn wrapWithPersistentChecksums(utilGenPackedInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])
-    dependsOn wrapWithPersistentChecksums(utilGenLevInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])
+    def extraConfig = [
+            andThenTasks: ["spotlessJava", "spotlessJavaApply"],
+            mustRunBefore: [ "compileJava" ]
+    ]
+
+    dependsOn wrapWithPersistentChecksums(utilGenPackedInternal, extraConfig)
+    dependsOn wrapWithPersistentChecksums(utilGenLevInternal, extraConfig)
   }
 
   regenerate.dependsOn moman

gradle/generation/regenerate.gradle

@@ -105,25 +105,33 @@ configure([
       //
       // The idea is that, given source-generating task "sourceTaskInternal" (note the suffix),
       // we create a bunch of other tasks that perform checksum generation, validation and
-      // source task skipping; example (sourceTask has Internal suffix stripped)
+      // source task skipping. For example, let's say we have a task 'genFooInternal";
+      // the following tasks would be created
       //
-      // ${sourceTask}ChecksumLoad
-      // ${sourceTask}ChecksumSave
-      // ${sourceTask}ChecksumCheck (fails if checksums are inconsistent)
-      // ${sourceTask} dependsOn [checksum-load, ${sourceTask}Internal, checksum-save]
+      // genFooChecksumLoad
+      // genFooChecksumSave
+      // genFooChecksumCheck (fails if checksums are inconsistent)
+      //
+      // and the following set of dependencies would be created (with additional
+      // constraints to run them in this particular order!).
+      //
+      // genFoo.dependsOn [genFooChecksumLoad, genFooInternal, genFooChecksumSave]
       //
       // Checksums are persisted and computed from sourceTask's inputs/outputs. If the
-      // persisted checksums are identical to current checksums, source task
-      // is skipped (via sourceTaskInternal.onlyIf { false }).
+      // persisted checksums are identical to the now-current checksums, the "internal" task
+      // is skipped (using onlyIf { false }).
       //
       // Implementation-wise things get complicated because gradle doesn't have the notion
       // of "ordered" task execution with respect to task AND its dependencies (we can add
       // constraints to each node in the execution graph but not node-and-dependencies).
       //
       // sourceTask - the task to wrap
       // extraConfig - a map with extra (optional) configuration options.
-      //   andThenTasks: other tasks that should be scheduled to run after source task and
-      //     before checksum calculation.
+      //   andThenTasks: any other tasks that should be scheduled to run after the internal task and
+      //     before checksum calculation/ saving. spotless is a good example of this.
+      //   ignoreWithSource: any other tasks that should be ignored if the internal task is ignored
+      //     (checksums are identical)
+      //   mustRunBefore: any tasks which should be scheduled to run after the internal task.
       wrapWithPersistentChecksums = { Task sourceTask, Map<String, Object> extraConfig = [:] ->
         if (!sourceTask.name.endsWith("Internal")) {
           throw new GradleException("Wrapped task must follow the convention name of *Internal: ${sourceTask.name}")
@@ -144,6 +152,13 @@ configure([
         List<Object> andThenTasks = toList(extraConfig.get("andThenTasks"))
         List<Object> ignoreWithSource =  toList(extraConfig.get("ignoreWithSource"))
 
+        // Schedule must-run-afters
+        List<Object> mustRunBefore = toList(extraConfig.get("mustRunBefore"))
+        // TODO: maybe ensure all task refs here are strings?
+        tasks.matching { it.name in mustRunBefore }.configureEach {
+          mustRunAfter sourceTask
+        }
+
         // Create checksum-loader task.
         Task checksumLoadTask = tasks.create("${sourceTaskName}ChecksumLoad", {
           ext {

gradle/generation/snowball.gradle

@@ -133,5 +133,9 @@ configure(project(":lucene:analysis:common")) {
     }
   }
 
-  regenerate.dependsOn wrapWithPersistentChecksums(snowballInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"], ignoreWithSource: [downloadSnowballStemmers, downloadSnowballWebsite, downloadSnowballData] ])
+  regenerate.dependsOn wrapWithPersistentChecksums(snowballInternal, [
+          andThenTasks: ["spotlessJava", "spotlessJavaApply"],
+          ignoreWithSource: [downloadSnowballStemmers, downloadSnowballWebsite, downloadSnowballData],
+          mustRunBefore: [ "compileJava" ]
+  ])
 }

gradle/generation/unicode-test-classes.gradle

@@ -43,7 +43,10 @@ configure(project(":lucene:test-framework")) {
       }
     }
   }
-  regenerate.dependsOn wrapWithPersistentChecksums(generateEmojiTokenizationTestInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])
+  regenerate.dependsOn wrapWithPersistentChecksums(generateEmojiTokenizationTestInternal, [
+          andThenTasks: ["spotlessJava", "spotlessJavaApply"],
+          mustRunBefore: [ "compileJava" ]
+  ])
 
   task generateJavaUnicodeWordBreakTestInternal() {
     def unicodeVersion = "12.1"
@@ -71,6 +74,8 @@ configure(project(":lucene:test-framework")) {
     }
   }
 
-  regenerate.dependsOn wrapWithPersistentChecksums(generateJavaUnicodeWordBreakTestInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])
-
+  regenerate.dependsOn wrapWithPersistentChecksums(generateJavaUnicodeWordBreakTestInternal, [
+          andThenTasks: ["spotlessJava", "spotlessJavaApply"],
+          mustRunBefore: [ "compileJava" ]
+  ])
 }

help/regeneration.txt

@@ -16,22 +16,6 @@ In many cases hitting (1) means you'll have to do (2) so let's discuss
 these in order.
 
 
-SPECIAL NOTE
-------------
-
-Regeneration tasks currently don't play well with the current gradle version.
-To workaround the issue, compile everything first:
-
-gradlew compileJava
-
-Then run the commands as described in this document, excluding compileJava.
-For example to run "regenerate":
-
-gradlew regenerate -x compileJava
-
-More information: https://github.com/apache/lucene/issues/13240
-
-
 Checksum validation errors
 --------------------------
 

lucene/licenses/opennlp-tools-1.9.4.jar.sha1

@@ -1 +1 @@
-dd5c4a6d82453bcccb78ba4ac90f166366dde12b
+dd5c4a6d82453bcccb78ba4ac90f166366dde12b