From d089cd672be9145e4183413943922fd4d38bfc7a Mon Sep 17 00:00:00 2001 From: Dawid Weiss Date: Thu, 6 Jun 2024 16:36:27 +0200 Subject: [PATCH] Backport generation-compilation ordering fixes from #13282. Correct whitespace difference in opennlp-tools-1.9.4.jar.sha1. --- gradle/generation/antlr.gradle | 5 ++- gradle/generation/forUtil.gradle | 15 +++++++-- gradle/generation/icu.gradle | 14 ++++++-- gradle/generation/javacc.gradle | 11 +++++-- gradle/generation/jflex.gradle | 26 +++++++++++---- gradle/generation/moman.gradle | 9 +++-- gradle/generation/regenerate.gradle | 33 ++++++++++++++----- gradle/generation/snowball.gradle | 6 +++- gradle/generation/unicode-test-classes.gradle | 11 +++++-- help/regeneration.txt | 16 --------- lucene/licenses/opennlp-tools-1.9.4.jar.sha1 | 2 +- 11 files changed, 99 insertions(+), 49 deletions(-) diff --git a/gradle/generation/antlr.gradle b/gradle/generation/antlr.gradle index 33992384fa1e..9e5d3fcb7c7a 100644 --- a/gradle/generation/antlr.gradle +++ b/gradle/generation/antlr.gradle @@ -82,7 +82,10 @@ configure(project(":lucene:expressions")) { } def generateAntlr = wrapWithPersistentChecksums( - generateAntlrInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ]) + generateAntlrInternal, [ + andThenTasks: ["spotlessJava", "spotlessJavaApply"], + mustRunBefore: [ "compileJava" ] + ]) regenerate.dependsOn generateAntlr } diff --git a/gradle/generation/forUtil.gradle b/gradle/generation/forUtil.gradle index 96f5bfdf9740..e14282348752 100644 --- a/gradle/generation/forUtil.gradle +++ b/gradle/generation/forUtil.gradle @@ -39,7 +39,10 @@ configure(project(":lucene:core")) { } } - regenerate.dependsOn wrapWithPersistentChecksums(generateForUtilInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ]) + regenerate.dependsOn wrapWithPersistentChecksums(generateForUtilInternal, [ + andThenTasks: ["spotlessJava", "spotlessJavaApply"], + mustRunBefore: [ "compileJava" ] + ]) } configure(project(":lucene:backward-codecs")) { @@ -64,7 +67,10 @@ configure(project(":lucene:backward-codecs")) { } } - regenerate.dependsOn wrapWithPersistentChecksums(generateForUtil84Internal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ]) + regenerate.dependsOn wrapWithPersistentChecksums(generateForUtil84Internal, [ + andThenTasks: ["spotlessJava", "spotlessJavaApply"], + mustRunBefore: [ "compileJava" ] + ]) task generateForUtil90Internal() { description "Regenerate gen_ForUtil.py" @@ -86,6 +92,9 @@ configure(project(":lucene:backward-codecs")) { } } - regenerate.dependsOn wrapWithPersistentChecksums(generateForUtil90Internal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ]) + regenerate.dependsOn wrapWithPersistentChecksums(generateForUtil90Internal, [ + andThenTasks: ["spotlessJava", "spotlessJavaApply"], + mustRunBefore: [ "compileJava" ] + ]) } diff --git a/gradle/generation/icu.gradle b/gradle/generation/icu.gradle index 85d9084b1df8..042f756bbdb5 100644 --- a/gradle/generation/icu.gradle +++ b/gradle/generation/icu.gradle @@ -159,8 +159,13 @@ configure(project(":lucene:analysis:icu")) { } } - regenerate.dependsOn wrapWithPersistentChecksums(genUtr30DataFilesInternal, [ ignoreWithSource: icuCompileTask ]) - regenerate.dependsOn wrapWithPersistentChecksums(genRbbiInternal) + regenerate.dependsOn wrapWithPersistentChecksums(genUtr30DataFilesInternal, [ + ignoreWithSource: icuCompileTask, + mustRunBefore: [ "compileJava" ] + ]) + regenerate.dependsOn wrapWithPersistentChecksums(genRbbiInternal, [ + mustRunBefore: [ "compileJava" ] + ]) task compileIcuWindows() { doFirst { @@ -281,5 +286,8 @@ configure(project(":lucene:analysis:common")) { } } - regenerate.dependsOn wrapWithPersistentChecksums(generateUnicodePropsInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ]) + regenerate.dependsOn wrapWithPersistentChecksums(generateUnicodePropsInternal, [ + andThenTasks: ["spotlessJava", "spotlessJavaApply"], + mustRunBefore: [ "compileJava" ] + ]) } diff --git a/gradle/generation/javacc.gradle b/gradle/generation/javacc.gradle index 3d0dd9161509..5409e3883a48 100644 --- a/gradle/generation/javacc.gradle +++ b/gradle/generation/javacc.gradle @@ -232,9 +232,14 @@ configure(project(":lucene:queryparser")) { description "Regenerate query parsers (javacc syntax definitions)." group "generation" - dependsOn wrapWithPersistentChecksums(javaccParserClassicInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ]), - wrapWithPersistentChecksums(javaccParserSurroundInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ]), - wrapWithPersistentChecksums(javaccParserFlexibleInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ]) + def extraConfig = [ + andThenTasks: ["spotlessJava", "spotlessJavaApply"], + mustRunBefore: [ "compileJava" ] + ] + + dependsOn wrapWithPersistentChecksums(javaccParserClassicInternal, extraConfig), + wrapWithPersistentChecksums(javaccParserSurroundInternal, extraConfig), + wrapWithPersistentChecksums(javaccParserFlexibleInternal, extraConfig) } regenerate.dependsOn javacc diff --git a/gradle/generation/jflex.gradle b/gradle/generation/jflex.gradle index c533c3cfa4fc..6c028b1279ef 100644 --- a/gradle/generation/jflex.gradle +++ b/gradle/generation/jflex.gradle @@ -51,7 +51,10 @@ configure(project(":lucene:core")) { } } - def generateStandardTokenizer = wrapWithPersistentChecksums(generateStandardTokenizerInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ]) + def generateStandardTokenizer = wrapWithPersistentChecksums(generateStandardTokenizerInternal, [ + andThenTasks: ["spotlessJava", "spotlessJavaApply"], + mustRunBefore: [ "compileJava" ] + ]) regenerate.dependsOn generateStandardTokenizer } @@ -149,7 +152,11 @@ configure(project(":lucene:analysis:common")) { } } - def generateUAX29URLEmailTokenizer = wrapWithPersistentChecksums(generateUAX29URLEmailTokenizerInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ]) + def generateUAX29URLEmailTokenizer = wrapWithPersistentChecksums(generateUAX29URLEmailTokenizerInternal, [ + andThenTasks: ["spotlessJava", "spotlessJavaApply"], + mustRunBefore: [ "compileJava" ] + ]) + // UAX29URLEmailTokenizerImpl.jflex includes: ASCIITLD.jflex // so we make sure it is up to date. Also, make sure the ordering of internal // tasks is enforced (just scheduling top-level wrappers doesn't mean their subtask graphs cannot be reordered). @@ -197,7 +204,12 @@ configure(project(":lucene:analysis:common")) { skeleton = skeletonDefault } - def generateHTMLStripCharFilter = wrapWithPersistentChecksums(generateHTMLStripCharFilterInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ]) + def extraConfig = [ + andThenTasks: ["spotlessJava", "spotlessJavaApply"], + mustRunBefore: [ "compileJava" ] + ] + + def generateHTMLStripCharFilter = wrapWithPersistentChecksums(generateHTMLStripCharFilterInternal, extraConfig) // HTMLStripCharFilter.jflex includes HTMLCharacterEntities.jflex so we make sure it's up to date. configure(generateHTMLStripCharFilter) { dependsOn "generateHTMLCharacterEntities" @@ -206,12 +218,12 @@ configure(project(":lucene:analysis:common")) { dependsOn "generateHTMLCharacterEntitiesInternal" } - regenerate.dependsOn wrapWithPersistentChecksums(generateWikipediaTokenizerInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ]), - wrapWithPersistentChecksums(generateClassicTokenizerInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ]), + regenerate.dependsOn wrapWithPersistentChecksums(generateWikipediaTokenizerInternal, extraConfig), + wrapWithPersistentChecksums(generateClassicTokenizerInternal, extraConfig), generateUAX29URLEmailTokenizer, - wrapWithPersistentChecksums(generateHTMLCharacterEntitiesInternal), + wrapWithPersistentChecksums(generateHTMLCharacterEntitiesInternal, extraConfig), generateHTMLStripCharFilter, - wrapWithPersistentChecksums(generateTldsInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ]) + wrapWithPersistentChecksums(generateTldsInternal, extraConfig) } class JFlexTask extends DefaultTask { diff --git a/gradle/generation/moman.gradle b/gradle/generation/moman.gradle index 62dbed15acf2..f825090c27c1 100644 --- a/gradle/generation/moman.gradle +++ b/gradle/generation/moman.gradle @@ -105,8 +105,13 @@ configure(project(":lucene:core")) { description "Regenerate Moman-based sources." group "generation" - dependsOn wrapWithPersistentChecksums(utilGenPackedInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ]) - dependsOn wrapWithPersistentChecksums(utilGenLevInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ]) + def extraConfig = [ + andThenTasks: ["spotlessJava", "spotlessJavaApply"], + mustRunBefore: [ "compileJava" ] + ] + + dependsOn wrapWithPersistentChecksums(utilGenPackedInternal, extraConfig) + dependsOn wrapWithPersistentChecksums(utilGenLevInternal, extraConfig) } regenerate.dependsOn moman diff --git a/gradle/generation/regenerate.gradle b/gradle/generation/regenerate.gradle index da438e5e14ea..06640a8619b1 100644 --- a/gradle/generation/regenerate.gradle +++ b/gradle/generation/regenerate.gradle @@ -105,16 +105,21 @@ configure([ // // The idea is that, given source-generating task "sourceTaskInternal" (note the suffix), // we create a bunch of other tasks that perform checksum generation, validation and - // source task skipping; example (sourceTask has Internal suffix stripped) + // source task skipping. For example, let's say we have a task 'genFooInternal"; + // the following tasks would be created // - // ${sourceTask}ChecksumLoad - // ${sourceTask}ChecksumSave - // ${sourceTask}ChecksumCheck (fails if checksums are inconsistent) - // ${sourceTask} dependsOn [checksum-load, ${sourceTask}Internal, checksum-save] + // genFooChecksumLoad + // genFooChecksumSave + // genFooChecksumCheck (fails if checksums are inconsistent) + // + // and the following set of dependencies would be created (with additional + // constraints to run them in this particular order!). + // + // genFoo.dependsOn [genFooChecksumLoad, genFooInternal, genFooChecksumSave] // // Checksums are persisted and computed from sourceTask's inputs/outputs. If the - // persisted checksums are identical to current checksums, source task - // is skipped (via sourceTaskInternal.onlyIf { false }). + // persisted checksums are identical to the now-current checksums, the "internal" task + // is skipped (using onlyIf { false }). // // Implementation-wise things get complicated because gradle doesn't have the notion // of "ordered" task execution with respect to task AND its dependencies (we can add @@ -122,8 +127,11 @@ configure([ // // sourceTask - the task to wrap // extraConfig - a map with extra (optional) configuration options. - // andThenTasks: other tasks that should be scheduled to run after source task and - // before checksum calculation. + // andThenTasks: any other tasks that should be scheduled to run after the internal task and + // before checksum calculation/ saving. spotless is a good example of this. + // ignoreWithSource: any other tasks that should be ignored if the internal task is ignored + // (checksums are identical) + // mustRunBefore: any tasks which should be scheduled to run after the internal task. wrapWithPersistentChecksums = { Task sourceTask, Map extraConfig = [:] -> if (!sourceTask.name.endsWith("Internal")) { throw new GradleException("Wrapped task must follow the convention name of *Internal: ${sourceTask.name}") @@ -144,6 +152,13 @@ configure([ List andThenTasks = toList(extraConfig.get("andThenTasks")) List ignoreWithSource = toList(extraConfig.get("ignoreWithSource")) + // Schedule must-run-afters + List mustRunBefore = toList(extraConfig.get("mustRunBefore")) + // TODO: maybe ensure all task refs here are strings? + tasks.matching { it.name in mustRunBefore }.configureEach { + mustRunAfter sourceTask + } + // Create checksum-loader task. Task checksumLoadTask = tasks.create("${sourceTaskName}ChecksumLoad", { ext { diff --git a/gradle/generation/snowball.gradle b/gradle/generation/snowball.gradle index aa1e5fbccdf4..378c621331eb 100644 --- a/gradle/generation/snowball.gradle +++ b/gradle/generation/snowball.gradle @@ -133,5 +133,9 @@ configure(project(":lucene:analysis:common")) { } } - regenerate.dependsOn wrapWithPersistentChecksums(snowballInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"], ignoreWithSource: [downloadSnowballStemmers, downloadSnowballWebsite, downloadSnowballData] ]) + regenerate.dependsOn wrapWithPersistentChecksums(snowballInternal, [ + andThenTasks: ["spotlessJava", "spotlessJavaApply"], + ignoreWithSource: [downloadSnowballStemmers, downloadSnowballWebsite, downloadSnowballData], + mustRunBefore: [ "compileJava" ] + ]) } diff --git a/gradle/generation/unicode-test-classes.gradle b/gradle/generation/unicode-test-classes.gradle index cd2086edeb16..0bf25aa190df 100644 --- a/gradle/generation/unicode-test-classes.gradle +++ b/gradle/generation/unicode-test-classes.gradle @@ -43,7 +43,10 @@ configure(project(":lucene:test-framework")) { } } } - regenerate.dependsOn wrapWithPersistentChecksums(generateEmojiTokenizationTestInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ]) + regenerate.dependsOn wrapWithPersistentChecksums(generateEmojiTokenizationTestInternal, [ + andThenTasks: ["spotlessJava", "spotlessJavaApply"], + mustRunBefore: [ "compileJava" ] + ]) task generateJavaUnicodeWordBreakTestInternal() { def unicodeVersion = "12.1" @@ -71,6 +74,8 @@ configure(project(":lucene:test-framework")) { } } - regenerate.dependsOn wrapWithPersistentChecksums(generateJavaUnicodeWordBreakTestInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ]) - + regenerate.dependsOn wrapWithPersistentChecksums(generateJavaUnicodeWordBreakTestInternal, [ + andThenTasks: ["spotlessJava", "spotlessJavaApply"], + mustRunBefore: [ "compileJava" ] + ]) } diff --git a/help/regeneration.txt b/help/regeneration.txt index 440acbf096d2..dc7ced3b988a 100644 --- a/help/regeneration.txt +++ b/help/regeneration.txt @@ -16,22 +16,6 @@ In many cases hitting (1) means you'll have to do (2) so let's discuss these in order. -SPECIAL NOTE ------------- - -Regeneration tasks currently don't play well with the current gradle version. -To workaround the issue, compile everything first: - -gradlew compileJava - -Then run the commands as described in this document, excluding compileJava. -For example to run "regenerate": - -gradlew regenerate -x compileJava - -More information: https://github.com/apache/lucene/issues/13240 - - Checksum validation errors -------------------------- diff --git a/lucene/licenses/opennlp-tools-1.9.4.jar.sha1 b/lucene/licenses/opennlp-tools-1.9.4.jar.sha1 index b367a761a448..fa7e85dd48fb 100644 --- a/lucene/licenses/opennlp-tools-1.9.4.jar.sha1 +++ b/lucene/licenses/opennlp-tools-1.9.4.jar.sha1 @@ -1 +1 @@ -dd5c4a6d82453bcccb78ba4ac90f166366dde12b \ No newline at end of file +dd5c4a6d82453bcccb78ba4ac90f166366dde12b