From e6daa41db3fc36b7d05983a1fab33626d5db2a32 Mon Sep 17 00:00:00 2001 From: Martin Wittlinger Date: Wed, 26 Oct 2022 20:41:49 +0200 Subject: [PATCH 01/13] feat: projectconfig --- buildSrc/build.gradle | 1 - github-bot/README.md | 62 ---------------- github-bot/build.gradle | 2 + .../laughing_train/api/ProjectGraphQL.java | 34 +++++++++ .../data/FindProjectConfigRequest.java | 7 ++ .../data/FindProjectConfigResult.java | 13 ++++ .../persistence/DataBaseMigration.java | 29 ++++++++ .../persistence/ProjectConfig.java | 50 +++++++++++++ .../services/ProjectConfigService.java | 36 ++++++++++ .../services/QodanaService.java | 70 ++++++++++++++++--- .../services/ServiceAdresses.java | 2 + .../src/main/resources/application.properties | 3 +- .../services/ProjectConfigServiceTest.java | 70 +++++++++++++++++++ 13 files changed, 305 insertions(+), 74 deletions(-) delete mode 100644 github-bot/README.md create mode 100644 github-bot/src/main/java/io/github/martinwitt/laughing_train/data/FindProjectConfigRequest.java create mode 100644 github-bot/src/main/java/io/github/martinwitt/laughing_train/data/FindProjectConfigResult.java create mode 100644 github-bot/src/main/java/io/github/martinwitt/laughing_train/persistence/ProjectConfig.java create mode 100644 github-bot/src/main/java/io/github/martinwitt/laughing_train/services/ProjectConfigService.java create mode 100644 github-bot/src/test/java/io/github/martinwitt/laughing_train/services/ProjectConfigServiceTest.java diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 3d636bbe7..6512eab56 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -21,5 +21,4 @@ repositories { implementation "org.kordamp.gradle:jandex-gradle-plugin:0.13.2" implementation "net.ltgt.gradle:gradle-errorprone-plugin:3.0.1" implementation 'io.smallrye:jandex:3.0.1' - testImplementation group: 'org.assertj', name: 'assertj-core', version: '3.23.1' } diff --git a/github-bot/README.md b/github-bot/README.md deleted file mode 100644 index 57af09f39..000000000 --- a/github-bot/README.md +++ /dev/null @@ -1,62 +0,0 @@ -# laughing_train Project - -This project uses Quarkus, the Supersonic Subatomic Java Framework. - -If you want to learn more about Quarkus, please visit its website: https://quarkus.io/ . - -## Running the application in dev mode - -You can run your application in dev mode that enables live coding using: -```shell script -./gradlew quarkusDev -``` - -> **_NOTE:_** Quarkus now ships with a Dev UI, which is available in dev mode only at http://localhost:8080/q/dev/. - -## Packaging and running the application - -The application can be packaged using: -```shell script -./gradlew build -``` -It produces the `quarkus-run.jar` file in the `build/quarkus-app/` directory. -Be aware that it’s not an _über-jar_ as the dependencies are copied into the `build/quarkus-app/lib/` directory. - -The application is now runnable using `java -jar build/quarkus-app/quarkus-run.jar`. - -If you want to build an _über-jar_, execute the following command: -```shell script -./gradlew build -Dquarkus.package.type=uber-jar -``` - -The application, packaged as an _über-jar_, is now runnable using `java -jar build/*-runner.jar`. - -## Creating a native executable - -You can create a native executable using: -```shell script -./gradlew build -Dquarkus.package.type=native -``` - -Or, if you don't have GraalVM installed, you can run the native executable build in a container using: -```shell script -./gradlew build -Dquarkus.package.type=native -Dquarkus.native.container-build=true -``` - -You can then execute your native executable with: `./build/laughing_train-0.0.1-runner` - -If you want to learn more about building native executables, please consult https://quarkus.io/guides/gradle-tooling. - -## Related Guides - -- GitHub API ([guide](https://github-api.kohsuke.org/)): Connect to the GitHub API -- GitHub App ([guide](https://quarkiverse.github.io/quarkiverse-docs/quarkus-github-app/dev/index.html)): Automate GitHub tasks with a GitHub App -- GitHub App Command Airline ([guide](https://quarkiverse.github.io/quarkiverse-docs/quarkus-github-app/dev/index.html)): Add comment-based commands to your GitHub App - - -package spoon.reflect.factory; - -package spoon.reflect.factory; - -aaaaa -package spoon.reflect.factory; diff --git a/github-bot/build.gradle b/github-bot/build.gradle index b2a39812e..f59e8a80a 100644 --- a/github-bot/build.gradle +++ b/github-bot/build.gradle @@ -27,6 +27,8 @@ dependencies { implementation 'io.quarkiverse.quinoa:quarkus-quinoa:1.2.2' implementation("io.quarkus:quarkus-oidc") implementation("io.quarkus:quarkus-keycloak-authorization") + testImplementation group: 'org.assertj', name: 'assertj-core', version: '3.23.1' + implementation 'io.vertx:vertx-junit5:+' } group 'io.github.martinwitt' diff --git a/github-bot/src/main/java/io/github/martinwitt/laughing_train/api/ProjectGraphQL.java b/github-bot/src/main/java/io/github/martinwitt/laughing_train/api/ProjectGraphQL.java index c40c31ec7..2b06b48b3 100644 --- a/github-bot/src/main/java/io/github/martinwitt/laughing_train/api/ProjectGraphQL.java +++ b/github-bot/src/main/java/io/github/martinwitt/laughing_train/api/ProjectGraphQL.java @@ -7,12 +7,17 @@ import static com.mongodb.client.model.Sorts.ascending; import com.google.common.flogger.FluentLogger; +import io.github.martinwitt.laughing_train.data.FindProjectConfigRequest; +import io.github.martinwitt.laughing_train.data.FindProjectConfigResult; import io.github.martinwitt.laughing_train.persistence.BadSmell; import io.github.martinwitt.laughing_train.persistence.Project; +import io.github.martinwitt.laughing_train.persistence.ProjectConfig; +import io.github.martinwitt.laughing_train.services.ProjectConfigService; import io.quarkus.security.Authenticated; import java.util.ArrayList; import java.util.List; import javax.enterprise.context.RequestScoped; +import javax.inject.Inject; import org.bson.BsonDocument; import org.bson.conversions.Bson; import org.eclipse.microprofile.graphql.DefaultValue; @@ -27,6 +32,9 @@ public class ProjectGraphQL { private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + @Inject + ProjectConfigService projectConfigService; + @Query("getProjects") @Description("Gets all projects from the database") public List getAllProjects() { @@ -77,4 +85,30 @@ public List removeProjectByName(String projectName) { public String login(@DefaultValue("defaultValue") String notNeeded) { return "login successful"; } + + @Query("getProjectConfig") + @Description("Gets the project config for a project") + public ProjectConfig getProjectConfig(String projectUrl) { + var result = projectConfigService.getConfig(new FindProjectConfigRequest.ByProjectUrl(projectUrl)); + if (result instanceof FindProjectConfigResult.SingleResult singleResult) { + return singleResult.projectConfig(); + } else { + return ProjectConfig.ofProjectUrl(projectUrl); + } + } + + @Mutation + @Authenticated + @Description("Sets the project config for a project") + public ProjectConfig setProjectConfig(ProjectConfig projectConfig) { + var result = projectConfigService.getConfig( + new FindProjectConfigRequest.ByProjectUrl(projectConfig.getProjectUrl())); + if (result instanceof FindProjectConfigResult.SingleResult singleResult) { + singleResult.projectConfig().update(projectConfig); + return singleResult.projectConfig(); + } else { + projectConfig.persist(); + return projectConfig; + } + } } diff --git a/github-bot/src/main/java/io/github/martinwitt/laughing_train/data/FindProjectConfigRequest.java b/github-bot/src/main/java/io/github/martinwitt/laughing_train/data/FindProjectConfigRequest.java new file mode 100644 index 000000000..4db1c8907 --- /dev/null +++ b/github-bot/src/main/java/io/github/martinwitt/laughing_train/data/FindProjectConfigRequest.java @@ -0,0 +1,7 @@ +package io.github.martinwitt.laughing_train.data; + +import java.io.Serializable; + +public sealed interface FindProjectConfigRequest extends Serializable { + record ByProjectUrl(String projectUrl) implements FindProjectConfigRequest, Serializable {} +} diff --git a/github-bot/src/main/java/io/github/martinwitt/laughing_train/data/FindProjectConfigResult.java b/github-bot/src/main/java/io/github/martinwitt/laughing_train/data/FindProjectConfigResult.java new file mode 100644 index 000000000..6e026fe80 --- /dev/null +++ b/github-bot/src/main/java/io/github/martinwitt/laughing_train/data/FindProjectConfigResult.java @@ -0,0 +1,13 @@ +package io.github.martinwitt.laughing_train.data; + +import io.github.martinwitt.laughing_train.persistence.ProjectConfig; +import java.io.Serializable; +import java.util.List; + +public sealed interface FindProjectConfigResult extends Serializable { + record SingleResult(ProjectConfig projectConfig) implements FindProjectConfigResult {} + + record MultipleResults(List projectConfigs) implements FindProjectConfigResult {} + + record NotFound() implements FindProjectConfigResult {} +} diff --git a/github-bot/src/main/java/io/github/martinwitt/laughing_train/persistence/DataBaseMigration.java b/github-bot/src/main/java/io/github/martinwitt/laughing_train/persistence/DataBaseMigration.java index cac8b89b6..8380c6f4a 100644 --- a/github-bot/src/main/java/io/github/martinwitt/laughing_train/persistence/DataBaseMigration.java +++ b/github-bot/src/main/java/io/github/martinwitt/laughing_train/persistence/DataBaseMigration.java @@ -4,6 +4,7 @@ import io.quarkus.mongodb.panache.PanacheMongoEntityBase; import io.quarkus.runtime.StartupEvent; import java.util.ArrayList; +import java.util.Map; import javax.enterprise.context.ApplicationScoped; import javax.enterprise.event.Observes; import org.apache.commons.lang3.StringUtils; @@ -21,6 +22,34 @@ public void onStart(@Observes StartupEvent event) { removeProjectHashesWithoutResults(); removeBadSmellsWithoutIdentifier(); removeBadSmellsWithWrongIdentifier(); + createConfigsIfMissing(); + setDefaultSourceFolders(); + } + + private void setDefaultSourceFolders() { + Map.of( + "https://github.com/google/guava", + "guava", + "https://github.com/INRIA/spoon", + "src/main/java", + "https://github.com/assertj/assertj", + "assertj-core") + .forEach((k, v) -> { + var list = ProjectConfig.findByProjectUrl(v); + if (list.size() == 1) { + var config = list.get(0); + config.setSourceFolder(k); + config.update(); + } + }); + } + + private void createConfigsIfMissing() { + Project.streamAll().forEach(project -> { + if (ProjectConfig.findByProjectUrl(project.getProjectUrl()).isEmpty()) { + ProjectConfig.ofProjectUrl(project.getProjectUrl()).persist(); + } + }); } private void removeBadSmellsWithWrongIdentifier() { diff --git a/github-bot/src/main/java/io/github/martinwitt/laughing_train/persistence/ProjectConfig.java b/github-bot/src/main/java/io/github/martinwitt/laughing_train/persistence/ProjectConfig.java new file mode 100644 index 000000000..7d1057f13 --- /dev/null +++ b/github-bot/src/main/java/io/github/martinwitt/laughing_train/persistence/ProjectConfig.java @@ -0,0 +1,50 @@ +package io.github.martinwitt.laughing_train.persistence; + +import io.quarkus.mongodb.panache.PanacheMongoEntity; +import io.quarkus.mongodb.panache.common.MongoEntity; +import java.io.Serializable; +import java.util.List; + +@MongoEntity(database = "Laughing-Train") +public class ProjectConfig extends PanacheMongoEntity implements Serializable { + + private String sourceFolder; + private String projectUrl; + + public ProjectConfig(String sourceFolder, String projectUrl) { + this.sourceFolder = sourceFolder; + this.projectUrl = projectUrl; + } + + public static ProjectConfig ofProjectUrl(String projectUrl) { + return new ProjectConfig(".", projectUrl); + } + + public ProjectConfig() { + sourceFolder = "."; + } + + public static List findByProjectUrl(String projectUrl) { + return find("projectUrl", projectUrl).list(); + } + + /** + * @param sourceFolder the sourceFolder to set + */ + public void setSourceFolder(String sourceFolder) { + this.sourceFolder = sourceFolder; + } + + /** + * @return the projectUrl + */ + public String getProjectUrl() { + return projectUrl; + } + /** + * @return the sourceFolder + */ + public String getSourceFolder() { + return sourceFolder; + } +} diff --git a/github-bot/src/main/java/io/github/martinwitt/laughing_train/services/ProjectConfigService.java b/github-bot/src/main/java/io/github/martinwitt/laughing_train/services/ProjectConfigService.java new file mode 100644 index 000000000..5c6183992 --- /dev/null +++ b/github-bot/src/main/java/io/github/martinwitt/laughing_train/services/ProjectConfigService.java @@ -0,0 +1,36 @@ +package io.github.martinwitt.laughing_train.services; + +import io.github.martinwitt.laughing_train.data.FindProjectConfigRequest; +import io.github.martinwitt.laughing_train.data.FindProjectConfigResult; +import io.github.martinwitt.laughing_train.persistence.ProjectConfig; +import io.quarkus.vertx.ConsumeEvent; +import io.smallrye.mutiny.Uni; +import java.util.List; +import javax.enterprise.context.ApplicationScoped; + +@ApplicationScoped +public class ProjectConfigService { + @ConsumeEvent(value = ServiceAdresses.PROJECT_CONFIG_REQUEST) + public FindProjectConfigResult getConfig(FindProjectConfigRequest request) { + if (request instanceof FindProjectConfigRequest.ByProjectUrl byProjectUrl) { + return Uni.createFrom() + .item(ProjectConfig.findByProjectUrl(byProjectUrl.projectUrl())) + .onItem() + .transform(this::convertToResult) + .await() + .indefinitely(); + } + return new FindProjectConfigResult.NotFound(); + } + + private FindProjectConfigResult convertToResult(List projectConfigs) { + if (projectConfigs.isEmpty()) { + return new FindProjectConfigResult.NotFound(); + } else if (projectConfigs.size() == 1) { + return new FindProjectConfigResult.SingleResult( + projectConfigs.iterator().next()); + } else { + return new FindProjectConfigResult.MultipleResults(projectConfigs); + } + } +} diff --git a/github-bot/src/main/java/io/github/martinwitt/laughing_train/services/QodanaService.java b/github-bot/src/main/java/io/github/martinwitt/laughing_train/services/QodanaService.java index e488f3b54..75036280a 100644 --- a/github-bot/src/main/java/io/github/martinwitt/laughing_train/services/QodanaService.java +++ b/github-bot/src/main/java/io/github/martinwitt/laughing_train/services/QodanaService.java @@ -4,9 +4,15 @@ import io.github.martinwitt.laughing_train.Config; import io.github.martinwitt.laughing_train.Constants; import io.github.martinwitt.laughing_train.data.AnalyzerRequest; +import io.github.martinwitt.laughing_train.data.FindProjectConfigRequest; +import io.github.martinwitt.laughing_train.data.FindProjectConfigResult; import io.github.martinwitt.laughing_train.data.QodanaResult; +import io.github.martinwitt.laughing_train.persistence.ProjectConfig; import io.quarkus.vertx.ConsumeEvent; +import io.vertx.core.AsyncResult; +import io.vertx.core.Future; import io.vertx.core.eventbus.EventBus; +import io.vertx.core.eventbus.Message; import java.io.Closeable; import java.io.IOException; import java.nio.file.Files; @@ -14,6 +20,7 @@ import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.function.Function; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import javax.enterprise.context.ApplicationScoped; @@ -74,16 +81,14 @@ public QodanaResult analyze(AnalyzerRequest request) { logger.atInfo().log("Received request %s", request); try { if (request instanceof AnalyzerRequest.WithProject project) { - var result = threadPoolManager - .getService() - .submit(() -> new QodanaResult.Success( - runQodana( - project.project().folder().toPath(), - project.project().sourceFolder()), - project.project())) - .get(); - eventBus.publish(ServiceAdresses.QODANA_ANALYZER_RESPONSE, result); - return result; + String projectUrl = project.project().url(); + return eventBus.request( + ServiceAdresses.PROJECT_CONFIG_REQUEST, + new FindProjectConfigRequest.ByProjectUrl(projectUrl)) + .transform(v -> transformToProjectResult(projectUrl, v)) + .transform(projectConfig -> tryInvokeQodana(project, projectConfig)) + .map(publishResults()) + .result(); } else { return new QodanaResult.Failure("Unknown request type"); } @@ -92,6 +97,51 @@ public QodanaResult analyze(AnalyzerRequest request) { } } + private Function publishResults() { + return result -> { + eventBus.publish(ServiceAdresses.QODANA_ANALYZER_RESPONSE, result); + return result; + }; + } + + private Future tryInvokeQodana( + AnalyzerRequest.WithProject project, AsyncResult projectConfig) { + if (projectConfig.succeeded()) { + return Future.succeededFuture(invokeQodana(project, projectConfig.result())); + } + return Future.failedFuture(projectConfig.cause()); + } + + private QodanaResult invokeQodana(AnalyzerRequest.WithProject project, ProjectConfig projectConfig) { + try { + return threadPoolManager + .getService() + .submit(() -> new QodanaResult.Success( + runQodana(project.project().folder().toPath(), projectConfig.getSourceFolder()), + project.project())) + .get(); + } catch (Exception e) { + return new QodanaResult.Failure(e.getMessage()); + } + } + + private Future transformToProjectResult( + String projectUrl, AsyncResult> v) { + if (v.failed()) { + logger.atSevere().withCause(v.cause()).log("Error while getting project config"); + return Future.failedFuture("Error while getting project config"); + } else { + FindProjectConfigResult result = v.result().body(); + if (result instanceof FindProjectConfigResult.SingleResult found) { + logger.atInfo().log("Found project config %s", found.projectConfig()); + return Future.succeededFuture(found.projectConfig()); + } else { + logger.atSevere().log("Could not find project config for %s", projectUrl); + return Future.failedFuture("Could not find project config for " + projectUrl); + } + } + } + private List runQodana(Path path, String sourceFolder) { QodanaAnalyzer analyzer = new QodanaAnalyzer.Builder() .withSourceFileRoot(sourceFolder) diff --git a/github-bot/src/main/java/io/github/martinwitt/laughing_train/services/ServiceAdresses.java b/github-bot/src/main/java/io/github/martinwitt/laughing_train/services/ServiceAdresses.java index a3219329c..1b512bdc3 100644 --- a/github-bot/src/main/java/io/github/martinwitt/laughing_train/services/ServiceAdresses.java +++ b/github-bot/src/main/java/io/github/martinwitt/laughing_train/services/ServiceAdresses.java @@ -9,4 +9,6 @@ private ServiceAdresses() {} public static final String PROJECT_RESPONSE = "project.response"; public static final String FIND_ISSUE_REQUEST = "github.issue.request"; public static final String FIND_SUMMARY_ISSUE_REQUEST = "github.issue-summary.request"; + public static final String PROJECT_CONFIG_REQUEST = "project.config.request"; + public static final String PROJECT_CONFIG_RESPONSE = "project.config.response"; } diff --git a/github-bot/src/main/resources/application.properties b/github-bot/src/main/resources/application.properties index 1897fa070..58c384660 100644 --- a/github-bot/src/main/resources/application.properties +++ b/github-bot/src/main/resources/application.properties @@ -45,4 +45,5 @@ pzwANJwZ0wzltgRGG9gpz2Lls3DJMRNZxvppL3wu74YKdr+kJxvbhjz0Z+304dCF\ YaGsVwKBgA6KSf1pPD93W/v5tV0WnhBZGKb3sX34JLQoCJPS5w6nTjHnuwpP9g0K\ C8HXKULXuX/rb/gz99PNcPQ9GqB0Rw/ho1KhO3VLJjBiSh/hiASKuFGs2y59UXYt\ mvqX3jAqi6ksGD5WPG6G2CdL/n30e5/lc10hq7SiwV7z9njW/8Pu\ ------END RSA PRIVATE KEY----- \ No newline at end of file +-----END RSA PRIVATE KEY----- +%dev.quarkus.scheduler.enabled=false \ No newline at end of file diff --git a/github-bot/src/test/java/io/github/martinwitt/laughing_train/services/ProjectConfigServiceTest.java b/github-bot/src/test/java/io/github/martinwitt/laughing_train/services/ProjectConfigServiceTest.java new file mode 100644 index 000000000..f02038251 --- /dev/null +++ b/github-bot/src/test/java/io/github/martinwitt/laughing_train/services/ProjectConfigServiceTest.java @@ -0,0 +1,70 @@ +package io.github.martinwitt.laughing_train.services; + +import static org.assertj.core.api.Assertions.assertThat; + +import io.github.martinwitt.laughing_train.data.FindProjectConfigRequest; +import io.github.martinwitt.laughing_train.data.FindProjectConfigResult; +import io.github.martinwitt.laughing_train.persistence.ProjectConfig; +import io.quarkus.test.junit.QuarkusTest; +import javax.inject.Inject; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +@QuarkusTest +public class ProjectConfigServiceTest { + + @Inject + ProjectConfigService projectConfigService; + + @AfterEach + void clearDataBase() { + ProjectConfig.deleteAll(); + } + + @Test + void emptyMessageReturnsNotfound() throws Throwable { + + FindProjectConfigRequest message = new FindProjectConfigRequest.ByProjectUrl("spoon.not.found"); + assertThat(projectConfigService.getConfig(message)).isInstanceOf(FindProjectConfigResult.NotFound.class); + } + + @Test + void defaultSrcFolderIsPoint() throws Throwable { + ProjectConfig config = ProjectConfig.ofProjectUrl("spoon.github.com"); + config.persist(); + FindProjectConfigRequest message = new FindProjectConfigRequest.ByProjectUrl("spoon.github.com"); + assertThat(projectConfigService.getConfig(message)).isInstanceOf(FindProjectConfigResult.SingleResult.class); + FindProjectConfigResult.SingleResult result = + (FindProjectConfigResult.SingleResult) projectConfigService.getConfig(message); + assertThat(result.projectConfig().getSourceFolder()).isEqualTo("."); + } + + @Test + void persistWithSourceFolder() throws Throwable { + ProjectConfig config = ProjectConfig.ofProjectUrl("spoon.github.com"); + config.setSourceFolder("src/main/java"); + config.persist(); + FindProjectConfigRequest message = new FindProjectConfigRequest.ByProjectUrl("spoon.github.com"); + assertThat(projectConfigService.getConfig(message)).isInstanceOf(FindProjectConfigResult.SingleResult.class); + FindProjectConfigResult.SingleResult result = + (FindProjectConfigResult.SingleResult) projectConfigService.getConfig(message); + assertThat(result.projectConfig().getSourceFolder()).isEqualTo("src/main/java"); + } + + @Test + void updateSourceFolder() throws Throwable { + ProjectConfig config = ProjectConfig.ofProjectUrl("spoon.github.com"); + config.setSourceFolder("src/main/java"); + config.persist(); + FindProjectConfigRequest message = new FindProjectConfigRequest.ByProjectUrl("spoon.github.com"); + assertThat(projectConfigService.getConfig(message)).isInstanceOf(FindProjectConfigResult.SingleResult.class); + FindProjectConfigResult.SingleResult result = + (FindProjectConfigResult.SingleResult) projectConfigService.getConfig(message); + assertThat(result.projectConfig().getSourceFolder()).isEqualTo("src/main/java"); + config.setSourceFolder("src/main/java2"); + config.persistOrUpdate(); + assertThat(projectConfigService.getConfig(message)).isInstanceOf(FindProjectConfigResult.SingleResult.class); + result = (FindProjectConfigResult.SingleResult) projectConfigService.getConfig(message); + assertThat(result.projectConfig().getSourceFolder()).isEqualTo("src/main/java2"); + } +} From affb3fb064ff07f137204773dde0089911875470 Mon Sep 17 00:00:00 2001 From: Martin Wittlinger Date: Wed, 26 Oct 2022 21:14:59 +0200 Subject: [PATCH 02/13] up --- .../services/QodanaService.java | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/github-bot/src/main/java/io/github/martinwitt/laughing_train/services/QodanaService.java b/github-bot/src/main/java/io/github/martinwitt/laughing_train/services/QodanaService.java index 75036280a..c16ed8422 100644 --- a/github-bot/src/main/java/io/github/martinwitt/laughing_train/services/QodanaService.java +++ b/github-bot/src/main/java/io/github/martinwitt/laughing_train/services/QodanaService.java @@ -11,6 +11,8 @@ import io.quarkus.vertx.ConsumeEvent; import io.vertx.core.AsyncResult; import io.vertx.core.Future; +import io.vertx.core.Vertx; +import io.vertx.core.eventbus.DeliveryOptions; import io.vertx.core.eventbus.EventBus; import io.vertx.core.eventbus.Message; import java.io.Closeable; @@ -20,6 +22,7 @@ import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.function.Function; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -44,6 +47,9 @@ public class QodanaService { @Inject EventBus eventBus; + @Inject + Vertx vertx; + public List runQodana(String gitUrl) throws IOException { Path file = Files.createTempDirectory(Constants.TEMP_FOLDER_PREFIX); try (Closeable closeable = () -> FileUtils.deleteDirectory(file.toFile())) { @@ -82,12 +88,15 @@ public QodanaResult analyze(AnalyzerRequest request) { try { if (request instanceof AnalyzerRequest.WithProject project) { String projectUrl = project.project().url(); - return eventBus.request( - ServiceAdresses.PROJECT_CONFIG_REQUEST, - new FindProjectConfigRequest.ByProjectUrl(projectUrl)) - .transform(v -> transformToProjectResult(projectUrl, v)) - .transform(projectConfig -> tryInvokeQodana(project, projectConfig)) - .map(publishResults()) + + return vertx.executeBlocking(value -> eventBus.request( + ServiceAdresses.PROJECT_CONFIG_REQUEST, + new FindProjectConfigRequest.ByProjectUrl(projectUrl), + new DeliveryOptions().setSendTimeout(TimeUnit.MINUTES.toMillis(30))) + .transform(v -> transformToProjectResult(projectUrl, v)) + .transform(projectConfig -> tryInvokeQodana(project, projectConfig)) + .map(publishResults()) + .result()) .result(); } else { return new QodanaResult.Failure("Unknown request type"); From 882debd427da3743a2582a1c65fc7137c5e38b71 Mon Sep 17 00:00:00 2001 From: Martin Wittlinger Date: Wed, 26 Oct 2022 21:42:26 +0200 Subject: [PATCH 03/13] up --- .../services/ProjectConfigService.java | 2 +- .../services/QodanaService.java | 21 ++++++++++--------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/github-bot/src/main/java/io/github/martinwitt/laughing_train/services/ProjectConfigService.java b/github-bot/src/main/java/io/github/martinwitt/laughing_train/services/ProjectConfigService.java index 5c6183992..4e5641476 100644 --- a/github-bot/src/main/java/io/github/martinwitt/laughing_train/services/ProjectConfigService.java +++ b/github-bot/src/main/java/io/github/martinwitt/laughing_train/services/ProjectConfigService.java @@ -10,7 +10,7 @@ @ApplicationScoped public class ProjectConfigService { - @ConsumeEvent(value = ServiceAdresses.PROJECT_CONFIG_REQUEST) + @ConsumeEvent(value = ServiceAdresses.PROJECT_CONFIG_REQUEST, blocking = true) public FindProjectConfigResult getConfig(FindProjectConfigRequest request) { if (request instanceof FindProjectConfigRequest.ByProjectUrl byProjectUrl) { return Uni.createFrom() diff --git a/github-bot/src/main/java/io/github/martinwitt/laughing_train/services/QodanaService.java b/github-bot/src/main/java/io/github/martinwitt/laughing_train/services/QodanaService.java index c16ed8422..a01340e26 100644 --- a/github-bot/src/main/java/io/github/martinwitt/laughing_train/services/QodanaService.java +++ b/github-bot/src/main/java/io/github/martinwitt/laughing_train/services/QodanaService.java @@ -87,16 +87,17 @@ public QodanaResult analyze(AnalyzerRequest request) { logger.atInfo().log("Received request %s", request); try { if (request instanceof AnalyzerRequest.WithProject project) { - String projectUrl = project.project().url(); - - return vertx.executeBlocking(value -> eventBus.request( - ServiceAdresses.PROJECT_CONFIG_REQUEST, - new FindProjectConfigRequest.ByProjectUrl(projectUrl), - new DeliveryOptions().setSendTimeout(TimeUnit.MINUTES.toMillis(30))) - .transform(v -> transformToProjectResult(projectUrl, v)) - .transform(projectConfig -> tryInvokeQodana(project, projectConfig)) - .map(publishResults()) - .result()) + return vertx.executeBlocking(promise -> { + String projectUrl = project.project().url(); + promise.complete(eventBus.request( + ServiceAdresses.PROJECT_CONFIG_REQUEST, + new FindProjectConfigRequest.ByProjectUrl(projectUrl), + new DeliveryOptions().setSendTimeout(TimeUnit.MINUTES.toMillis(30))) + .transform(v -> transformToProjectResult(projectUrl, v)) + .transform(projectConfig -> tryInvokeQodana(project, projectConfig)) + .map(publishResults()) + .result()); + }) .result(); } else { return new QodanaResult.Failure("Unknown request type"); From 5b77f7611e048a44958d37fca7d482c8e542b99f Mon Sep 17 00:00:00 2001 From: Martin Wittlinger Date: Wed, 26 Oct 2022 22:54:45 +0200 Subject: [PATCH 04/13] up --- .../laughing_train/api/ProjectGraphQL.java | 3 ++ .../services/QodanaService.java | 54 +++++++++++-------- 2 files changed, 35 insertions(+), 22 deletions(-) diff --git a/github-bot/src/main/java/io/github/martinwitt/laughing_train/api/ProjectGraphQL.java b/github-bot/src/main/java/io/github/martinwitt/laughing_train/api/ProjectGraphQL.java index 2b06b48b3..1df85848b 100644 --- a/github-bot/src/main/java/io/github/martinwitt/laughing_train/api/ProjectGraphQL.java +++ b/github-bot/src/main/java/io/github/martinwitt/laughing_train/api/ProjectGraphQL.java @@ -66,6 +66,9 @@ public List getHashesForProject(String projectName) { @Description("Adds a project to the database") public Project addProject(String projectUrl, String projectName) { Project project = new Project(projectName, projectUrl); + if (ProjectConfig.findByProjectUrl(projectUrl).isEmpty()) { + ProjectConfig.ofProjectUrl(projectUrl).persistOrUpdate(); + } project.persistOrUpdate(); return project; } diff --git a/github-bot/src/main/java/io/github/martinwitt/laughing_train/services/QodanaService.java b/github-bot/src/main/java/io/github/martinwitt/laughing_train/services/QodanaService.java index a01340e26..44209a1b1 100644 --- a/github-bot/src/main/java/io/github/martinwitt/laughing_train/services/QodanaService.java +++ b/github-bot/src/main/java/io/github/martinwitt/laughing_train/services/QodanaService.java @@ -23,7 +23,6 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; -import java.util.function.Function; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import javax.enterprise.context.ApplicationScoped; @@ -89,14 +88,36 @@ public QodanaResult analyze(AnalyzerRequest request) { if (request instanceof AnalyzerRequest.WithProject project) { return vertx.executeBlocking(promise -> { String projectUrl = project.project().url(); - promise.complete(eventBus.request( - ServiceAdresses.PROJECT_CONFIG_REQUEST, - new FindProjectConfigRequest.ByProjectUrl(projectUrl), - new DeliveryOptions().setSendTimeout(TimeUnit.MINUTES.toMillis(30))) - .transform(v -> transformToProjectResult(projectUrl, v)) - .transform(projectConfig -> tryInvokeQodana(project, projectConfig)) - .map(publishResults()) - .result()); + eventBus.request( + ServiceAdresses.PROJECT_CONFIG_REQUEST, + new FindProjectConfigRequest.ByProjectUrl(projectUrl), + new DeliveryOptions().setSendTimeout(TimeUnit.MINUTES.toMillis(30)), + handler -> { + if (handler.succeeded()) { + FindProjectConfigResult result = + handler.result().body(); + if (result instanceof FindProjectConfigResult.SingleResult found) { + ProjectConfig projectConfig = found.projectConfig(); + logger.atInfo().log("Found config %s", projectConfig); + try { + QodanaResult results = invokeQodana(project, projectConfig); + if (results instanceof QodanaResult.Success success) { + promise.complete(new QodanaResult.Success( + success.result(), project.project())); + publishResults(success); + } else { + promise.fail("Qodana failed"); + } + } catch (Exception e) { + logger.atSevere().withCause(e).log("Error while running qodana"); + promise.complete(new QodanaResult.Failure(e.getMessage())); + } + } else { + logger.atSevere().log("No config found for %s", projectUrl); + promise.complete(new QodanaResult.Failure("No config found")); + } + } + }); }) .result(); } else { @@ -107,19 +128,8 @@ public QodanaResult analyze(AnalyzerRequest request) { } } - private Function publishResults() { - return result -> { - eventBus.publish(ServiceAdresses.QODANA_ANALYZER_RESPONSE, result); - return result; - }; - } - - private Future tryInvokeQodana( - AnalyzerRequest.WithProject project, AsyncResult projectConfig) { - if (projectConfig.succeeded()) { - return Future.succeededFuture(invokeQodana(project, projectConfig.result())); - } - return Future.failedFuture(projectConfig.cause()); + private void publishResults(QodanaResult result) { + eventBus.publish(ServiceAdresses.QODANA_ANALYZER_RESPONSE, result); } private QodanaResult invokeQodana(AnalyzerRequest.WithProject project, ProjectConfig projectConfig) { From 75c39dc61131a04dc7f47b2f846741ae2f8878d7 Mon Sep 17 00:00:00 2001 From: Martin Wittlinger Date: Thu, 27 Oct 2022 00:29:27 +0200 Subject: [PATCH 05/13] up --- .../laughing_train/mining/PeriodicMiner.java | 2 +- .../services/QodanaService.java | 79 ++++++------------- 2 files changed, 24 insertions(+), 57 deletions(-) diff --git a/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/PeriodicMiner.java b/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/PeriodicMiner.java index 9b75a867d..27a3fb638 100644 --- a/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/PeriodicMiner.java +++ b/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/PeriodicMiner.java @@ -45,7 +45,7 @@ public class PeriodicMiner { @Inject Vertx vertx; - @Scheduled(every = "4h", delay = 10, delayUnit = TimeUnit.MINUTES) + @Scheduled(every = "4h", delay = 3, delayUnit = TimeUnit.MINUTES) void mineRepos() { for (Project project : Project.findAll().list()) { eventBus.request( diff --git a/github-bot/src/main/java/io/github/martinwitt/laughing_train/services/QodanaService.java b/github-bot/src/main/java/io/github/martinwitt/laughing_train/services/QodanaService.java index 44209a1b1..ff1cf1058 100644 --- a/github-bot/src/main/java/io/github/martinwitt/laughing_train/services/QodanaService.java +++ b/github-bot/src/main/java/io/github/martinwitt/laughing_train/services/QodanaService.java @@ -4,17 +4,14 @@ import io.github.martinwitt.laughing_train.Config; import io.github.martinwitt.laughing_train.Constants; import io.github.martinwitt.laughing_train.data.AnalyzerRequest; +import io.github.martinwitt.laughing_train.data.AnalyzerRequest.WithProject; import io.github.martinwitt.laughing_train.data.FindProjectConfigRequest; import io.github.martinwitt.laughing_train.data.FindProjectConfigResult; import io.github.martinwitt.laughing_train.data.QodanaResult; import io.github.martinwitt.laughing_train.persistence.ProjectConfig; import io.quarkus.vertx.ConsumeEvent; -import io.vertx.core.AsyncResult; -import io.vertx.core.Future; import io.vertx.core.Vertx; -import io.vertx.core.eventbus.DeliveryOptions; import io.vertx.core.eventbus.EventBus; -import io.vertx.core.eventbus.Message; import java.io.Closeable; import java.io.IOException; import java.nio.file.Files; @@ -22,7 +19,6 @@ import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import javax.enterprise.context.ApplicationScoped; @@ -49,6 +45,9 @@ public class QodanaService { @Inject Vertx vertx; + @Inject + ProjectConfigService projectConfigService; + public List runQodana(String gitUrl) throws IOException { Path file = Files.createTempDirectory(Constants.TEMP_FOLDER_PREFIX); try (Closeable closeable = () -> FileUtils.deleteDirectory(file.toFile())) { @@ -86,40 +85,7 @@ public QodanaResult analyze(AnalyzerRequest request) { logger.atInfo().log("Received request %s", request); try { if (request instanceof AnalyzerRequest.WithProject project) { - return vertx.executeBlocking(promise -> { - String projectUrl = project.project().url(); - eventBus.request( - ServiceAdresses.PROJECT_CONFIG_REQUEST, - new FindProjectConfigRequest.ByProjectUrl(projectUrl), - new DeliveryOptions().setSendTimeout(TimeUnit.MINUTES.toMillis(30)), - handler -> { - if (handler.succeeded()) { - FindProjectConfigResult result = - handler.result().body(); - if (result instanceof FindProjectConfigResult.SingleResult found) { - ProjectConfig projectConfig = found.projectConfig(); - logger.atInfo().log("Found config %s", projectConfig); - try { - QodanaResult results = invokeQodana(project, projectConfig); - if (results instanceof QodanaResult.Success success) { - promise.complete(new QodanaResult.Success( - success.result(), project.project())); - publishResults(success); - } else { - promise.fail("Qodana failed"); - } - } catch (Exception e) { - logger.atSevere().withCause(e).log("Error while running qodana"); - promise.complete(new QodanaResult.Failure(e.getMessage())); - } - } else { - logger.atSevere().log("No config found for %s", projectUrl); - promise.complete(new QodanaResult.Failure("No config found")); - } - } - }); - }) - .result(); + return runQodanaWithConfig(project); } else { return new QodanaResult.Failure("Unknown request type"); } @@ -128,6 +94,24 @@ public QodanaResult analyze(AnalyzerRequest request) { } } + private QodanaResult runQodanaWithConfig(AnalyzerRequest.WithProject project) { + var projectConfig = getProjectConfig(project); + if (projectConfig instanceof FindProjectConfigResult.SingleResult found) { + var qodanaResult = invokeQodana(project, found.projectConfig()); + if (qodanaResult instanceof QodanaResult.Success success) { + publishResults(success); + } + return qodanaResult; + } else { + return new QodanaResult.Failure("No config found"); + } + } + + private FindProjectConfigResult getProjectConfig(WithProject item) { + return projectConfigService.getConfig( + new FindProjectConfigRequest.ByProjectUrl(item.project().url())); + } + private void publishResults(QodanaResult result) { eventBus.publish(ServiceAdresses.QODANA_ANALYZER_RESPONSE, result); } @@ -145,23 +129,6 @@ private QodanaResult invokeQodana(AnalyzerRequest.WithProject project, ProjectCo } } - private Future transformToProjectResult( - String projectUrl, AsyncResult> v) { - if (v.failed()) { - logger.atSevere().withCause(v.cause()).log("Error while getting project config"); - return Future.failedFuture("Error while getting project config"); - } else { - FindProjectConfigResult result = v.result().body(); - if (result instanceof FindProjectConfigResult.SingleResult found) { - logger.atInfo().log("Found project config %s", found.projectConfig()); - return Future.succeededFuture(found.projectConfig()); - } else { - logger.atSevere().log("Could not find project config for %s", projectUrl); - return Future.failedFuture("Could not find project config for " + projectUrl); - } - } - } - private List runQodana(Path path, String sourceFolder) { QodanaAnalyzer analyzer = new QodanaAnalyzer.Builder() .withSourceFileRoot(sourceFolder) From a3e1e91c414ef30b4e0d67951151d63f4bd6d140 Mon Sep 17 00:00:00 2001 From: Martin Wittlinger Date: Thu, 27 Oct 2022 00:57:53 +0200 Subject: [PATCH 06/13] more logging --- .../analyzer/qodana/rules/UnnecessaryToStringCall.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/code-transformation/src/main/java/xyz/keksdose/spoon/code_solver/analyzer/qodana/rules/UnnecessaryToStringCall.java b/code-transformation/src/main/java/xyz/keksdose/spoon/code_solver/analyzer/qodana/rules/UnnecessaryToStringCall.java index b61fe4d0d..3bab28b55 100644 --- a/code-transformation/src/main/java/xyz/keksdose/spoon/code_solver/analyzer/qodana/rules/UnnecessaryToStringCall.java +++ b/code-transformation/src/main/java/xyz/keksdose/spoon/code_solver/analyzer/qodana/rules/UnnecessaryToStringCall.java @@ -42,7 +42,8 @@ public void refactor(ChangeListener listener, CtType type) { return; } for (CtInvocation toStringInvocation : - filterMatches(PositionScanner.findLineOnly(type, result.position()))) { + filterMatches(PositionScanner.findLineOnly(type, result.position()))) { + logger.atInfo().log("found %s", toStringInvocation); CtInvocation oldInvocation = toStringInvocation.clone().setParent(null); toStringInvocation.replace(toStringInvocation.getTarget().clone()); listener.setChanged( From 3d79e03f991ad546f5e7247b4dde07a11012ee27 Mon Sep 17 00:00:00 2001 From: Martin Wittlinger Date: Thu, 27 Oct 2022 01:07:23 +0200 Subject: [PATCH 07/13] add caching --- .../code_solver/analyzer/qodana/QodanaAnalyzer.java | 9 +++++++++ .../analyzer/qodana/rules/UnnecessaryToStringCall.java | 2 +- .../laughing_train/services/QodanaService.java | 1 + 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/code-transformation/src/main/java/xyz/keksdose/spoon/code_solver/analyzer/qodana/QodanaAnalyzer.java b/code-transformation/src/main/java/xyz/keksdose/spoon/code_solver/analyzer/qodana/QodanaAnalyzer.java index 35a3dee20..062d514fa 100644 --- a/code-transformation/src/main/java/xyz/keksdose/spoon/code_solver/analyzer/qodana/QodanaAnalyzer.java +++ b/code-transformation/src/main/java/xyz/keksdose/spoon/code_solver/analyzer/qodana/QodanaAnalyzer.java @@ -183,6 +183,15 @@ private HostConfig createHostConfig(Path sourceRoot) { Volume targetFile = new Volume("/data/results/"); Bind bind = new Bind(sourceRoot.toAbsolutePath().toString(), sourceFile, AccessMode.rw); Bind resultsBind = new Bind(Path.of(resultFolder).toAbsolutePath().toString(), targetFile, AccessMode.rw); + if (qodanaCache.isPresent()) { + Volume cacheFile = new Volume("/data/cache/"); + Bind cacheBind = + new Bind(Path.of(qodanaCache.get()).toAbsolutePath().toString(), cacheFile, AccessMode.rw); + return HostConfig.newHostConfig() + .withBinds(bind, resultsBind, cacheBind) + .withPrivileged(true) + .withAutoRemove(true); + } return HostConfig.newHostConfig() .withBinds(bind, resultsBind) .withPrivileged(true) diff --git a/code-transformation/src/main/java/xyz/keksdose/spoon/code_solver/analyzer/qodana/rules/UnnecessaryToStringCall.java b/code-transformation/src/main/java/xyz/keksdose/spoon/code_solver/analyzer/qodana/rules/UnnecessaryToStringCall.java index 3bab28b55..e5e631e91 100644 --- a/code-transformation/src/main/java/xyz/keksdose/spoon/code_solver/analyzer/qodana/rules/UnnecessaryToStringCall.java +++ b/code-transformation/src/main/java/xyz/keksdose/spoon/code_solver/analyzer/qodana/rules/UnnecessaryToStringCall.java @@ -42,7 +42,7 @@ public void refactor(ChangeListener listener, CtType type) { return; } for (CtInvocation toStringInvocation : - filterMatches(PositionScanner.findLineOnly(type, result.position()))) { + filterMatches(PositionScanner.findLineOnly(type, result.position()))) { logger.atInfo().log("found %s", toStringInvocation); CtInvocation oldInvocation = toStringInvocation.clone().setParent(null); toStringInvocation.replace(toStringInvocation.getTarget().clone()); diff --git a/github-bot/src/main/java/io/github/martinwitt/laughing_train/services/QodanaService.java b/github-bot/src/main/java/io/github/martinwitt/laughing_train/services/QodanaService.java index ff1cf1058..c1ec8a500 100644 --- a/github-bot/src/main/java/io/github/martinwitt/laughing_train/services/QodanaService.java +++ b/github-bot/src/main/java/io/github/martinwitt/laughing_train/services/QodanaService.java @@ -132,6 +132,7 @@ private QodanaResult invokeQodana(AnalyzerRequest.WithProject project, ProjectCo private List runQodana(Path path, String sourceFolder) { QodanaAnalyzer analyzer = new QodanaAnalyzer.Builder() .withSourceFileRoot(sourceFolder) + .withCacheVolume("lauging-train.qodana-cache") .withResultFolder(path.toAbsolutePath().toString()) .build(); return analyzer.runQodana(path); From 25e83c5a8fb56a4c6cb0bf1d21481f4968ae9d7b Mon Sep 17 00:00:00 2001 From: Martin Wittlinger Date: Thu, 27 Oct 2022 02:08:35 +0200 Subject: [PATCH 08/13] try cache subfolder --- .../code_solver/analyzer/qodana/QodanaAnalyzer.java | 10 +++++++--- .../laughing_train/services/QodanaService.java | 9 ++++++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/code-transformation/src/main/java/xyz/keksdose/spoon/code_solver/analyzer/qodana/QodanaAnalyzer.java b/code-transformation/src/main/java/xyz/keksdose/spoon/code_solver/analyzer/qodana/QodanaAnalyzer.java index 062d514fa..4609290e8 100644 --- a/code-transformation/src/main/java/xyz/keksdose/spoon/code_solver/analyzer/qodana/QodanaAnalyzer.java +++ b/code-transformation/src/main/java/xyz/keksdose/spoon/code_solver/analyzer/qodana/QodanaAnalyzer.java @@ -45,6 +45,7 @@ public class QodanaAnalyzer { private String resultPathString; private String sourceFileRoot; private Optional qodanaCache; + private String cacheSubFolder; private QodanaAnalyzer(Builder builder) { this.resultFolder = builder.resultFolder; @@ -52,6 +53,7 @@ private QodanaAnalyzer(Builder builder) { this.resultPathString = builder.resultPathString; this.sourceFileRoot = builder.sourceFileRoot; this.qodanaCache = builder.cacheFolder; + this.cacheSubFolder = builder.subFolder; } public List runQodana(Path sourceRoot) { @@ -185,8 +187,8 @@ private HostConfig createHostConfig(Path sourceRoot) { Bind resultsBind = new Bind(Path.of(resultFolder).toAbsolutePath().toString(), targetFile, AccessMode.rw); if (qodanaCache.isPresent()) { Volume cacheFile = new Volume("/data/cache/"); - Bind cacheBind = - new Bind(Path.of(qodanaCache.get()).toAbsolutePath().toString(), cacheFile, AccessMode.rw); + Bind cacheBind = new Bind( + Path.of(qodanaCache.get()).toAbsolutePath().toString() + cacheSubFolder, cacheFile, AccessMode.rw); return HostConfig.newHostConfig() .withBinds(bind, resultsBind, cacheBind) .withPrivileged(true) @@ -282,6 +284,7 @@ public static class Builder { private String resultPathString = resultFolder + "/qodana.sarif.json"; private String sourceFileRoot = "./src/main/java"; private Optional cacheFolder = Optional.empty(); + private String subFolder; public Builder withResultFolder(String resultFolder) { this.resultFolder = resultFolder; @@ -306,8 +309,9 @@ public Builder withSourceFileRoot(String sourceFileRoot) { return this; } - public Builder withCacheVolume(String volumeName) { + public Builder withCacheVolume(String volumeName, String subFolder) { this.cacheFolder = Optional.of(volumeName); + this.subFolder = subFolder; return this; } diff --git a/github-bot/src/main/java/io/github/martinwitt/laughing_train/services/QodanaService.java b/github-bot/src/main/java/io/github/martinwitt/laughing_train/services/QodanaService.java index c1ec8a500..d44869e00 100644 --- a/github-bot/src/main/java/io/github/martinwitt/laughing_train/services/QodanaService.java +++ b/github-bot/src/main/java/io/github/martinwitt/laughing_train/services/QodanaService.java @@ -121,7 +121,10 @@ private QodanaResult invokeQodana(AnalyzerRequest.WithProject project, ProjectCo return threadPoolManager .getService() .submit(() -> new QodanaResult.Success( - runQodana(project.project().folder().toPath(), projectConfig.getSourceFolder()), + runQodana( + project.project().folder().toPath(), + projectConfig.getSourceFolder(), + project.project().name()), project.project())) .get(); } catch (Exception e) { @@ -129,10 +132,10 @@ private QodanaResult invokeQodana(AnalyzerRequest.WithProject project, ProjectCo } } - private List runQodana(Path path, String sourceFolder) { + private List runQodana(Path path, String sourceFolder, String projectName) { QodanaAnalyzer analyzer = new QodanaAnalyzer.Builder() .withSourceFileRoot(sourceFolder) - .withCacheVolume("lauging-train.qodana-cache") + .withCacheVolume("lauging-train.qodana-cache", projectName) .withResultFolder(path.toAbsolutePath().toString()) .build(); return analyzer.runQodana(path); From 40238e53594ef4bfcf488e9e654e1ca257dbf213 Mon Sep 17 00:00:00 2001 From: Martin Wittlinger Date: Thu, 27 Oct 2022 02:50:40 +0200 Subject: [PATCH 09/13] fix cache --- .../spoon/code_solver/analyzer/qodana/QodanaAnalyzer.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/code-transformation/src/main/java/xyz/keksdose/spoon/code_solver/analyzer/qodana/QodanaAnalyzer.java b/code-transformation/src/main/java/xyz/keksdose/spoon/code_solver/analyzer/qodana/QodanaAnalyzer.java index 4609290e8..f889dde09 100644 --- a/code-transformation/src/main/java/xyz/keksdose/spoon/code_solver/analyzer/qodana/QodanaAnalyzer.java +++ b/code-transformation/src/main/java/xyz/keksdose/spoon/code_solver/analyzer/qodana/QodanaAnalyzer.java @@ -176,7 +176,7 @@ private CreateContainerResponse createQodanaContainer( .withHostConfig(hostConfig) .withAttachStderr(true) .withAttachStdout(true) - .withCmd("-d", sourceFileRoot) + .withCmd("-d", sourceFileRoot, "--cache-dir=", qodanaCache.get() + "/" + cacheSubFolder) .exec(); } @@ -187,8 +187,8 @@ private HostConfig createHostConfig(Path sourceRoot) { Bind resultsBind = new Bind(Path.of(resultFolder).toAbsolutePath().toString(), targetFile, AccessMode.rw); if (qodanaCache.isPresent()) { Volume cacheFile = new Volume("/data/cache/"); - Bind cacheBind = new Bind( - Path.of(qodanaCache.get()).toAbsolutePath().toString() + cacheSubFolder, cacheFile, AccessMode.rw); + Bind cacheBind = + new Bind(Path.of(qodanaCache.get()).toAbsolutePath().toString(), cacheFile, AccessMode.rw); return HostConfig.newHostConfig() .withBinds(bind, resultsBind, cacheBind) .withPrivileged(true) @@ -283,7 +283,7 @@ public static class Builder { private String qodanaImageName = "jetbrains/qodana"; private String resultPathString = resultFolder + "/qodana.sarif.json"; private String sourceFileRoot = "./src/main/java"; - private Optional cacheFolder = Optional.empty(); + private Optional cacheFolder = Optional.of("data/cache"); private String subFolder; public Builder withResultFolder(String resultFolder) { From 8ba30a374decc22539f2d5286679f068a3688bdd Mon Sep 17 00:00:00 2001 From: Martin Wittlinger Date: Thu, 27 Oct 2022 17:39:02 +0200 Subject: [PATCH 10/13] remove broken cache --- .../analyzer/qodana/QodanaAnalyzer.java | 15 +-------------- .../laughing_train/mining/PeriodicMiner.java | 16 ++++------------ .../laughing_train/services/QodanaService.java | 8 ++------ 3 files changed, 7 insertions(+), 32 deletions(-) diff --git a/code-transformation/src/main/java/xyz/keksdose/spoon/code_solver/analyzer/qodana/QodanaAnalyzer.java b/code-transformation/src/main/java/xyz/keksdose/spoon/code_solver/analyzer/qodana/QodanaAnalyzer.java index f889dde09..b74204dba 100644 --- a/code-transformation/src/main/java/xyz/keksdose/spoon/code_solver/analyzer/qodana/QodanaAnalyzer.java +++ b/code-transformation/src/main/java/xyz/keksdose/spoon/code_solver/analyzer/qodana/QodanaAnalyzer.java @@ -44,16 +44,12 @@ public class QodanaAnalyzer { private String qodanaImageName; private String resultPathString; private String sourceFileRoot; - private Optional qodanaCache; - private String cacheSubFolder; private QodanaAnalyzer(Builder builder) { this.resultFolder = builder.resultFolder; this.qodanaImageName = builder.qodanaImageName; this.resultPathString = builder.resultPathString; this.sourceFileRoot = builder.sourceFileRoot; - this.qodanaCache = builder.cacheFolder; - this.cacheSubFolder = builder.subFolder; } public List runQodana(Path sourceRoot) { @@ -176,7 +172,7 @@ private CreateContainerResponse createQodanaContainer( .withHostConfig(hostConfig) .withAttachStderr(true) .withAttachStdout(true) - .withCmd("-d", sourceFileRoot, "--cache-dir=", qodanaCache.get() + "/" + cacheSubFolder) + .withCmd("-d", sourceFileRoot) .exec(); } @@ -185,15 +181,6 @@ private HostConfig createHostConfig(Path sourceRoot) { Volume targetFile = new Volume("/data/results/"); Bind bind = new Bind(sourceRoot.toAbsolutePath().toString(), sourceFile, AccessMode.rw); Bind resultsBind = new Bind(Path.of(resultFolder).toAbsolutePath().toString(), targetFile, AccessMode.rw); - if (qodanaCache.isPresent()) { - Volume cacheFile = new Volume("/data/cache/"); - Bind cacheBind = - new Bind(Path.of(qodanaCache.get()).toAbsolutePath().toString(), cacheFile, AccessMode.rw); - return HostConfig.newHostConfig() - .withBinds(bind, resultsBind, cacheBind) - .withPrivileged(true) - .withAutoRemove(true); - } return HostConfig.newHostConfig() .withBinds(bind, resultsBind) .withPrivileged(true) diff --git a/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/PeriodicMiner.java b/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/PeriodicMiner.java index 27a3fb638..86826b048 100644 --- a/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/PeriodicMiner.java +++ b/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/PeriodicMiner.java @@ -47,7 +47,10 @@ public class PeriodicMiner { @Scheduled(every = "4h", delay = 3, delayUnit = TimeUnit.MINUTES) void mineRepos() { - for (Project project : Project.findAll().list()) { + // for (Project project : Project.findAll().list()) { + for (Project project : Project.findAll().stream() + .filter(v -> v.getProjectName().contains("guava")) + .toList()) { eventBus.request( ServiceAdresses.PROJECT_REQUEST, new ProjectRequest.WithUrl(project.getProjectUrl()), @@ -62,7 +65,6 @@ private Promise mineProject(String repoName, AsyncResult query = Project.findByProjectName(project.project().name()); - removeOldDbEntry(query); if (query.size() == 1) { query.get(0).addCommitHash(project.project().commitHash()); query.get(0).persistOrUpdate(); @@ -87,16 +89,6 @@ private Promise mineProject(String repoName, AsyncResult query) { - query.stream().filter(v -> v.getCommitHashes() == null).forEach(v -> v.delete()); - query.stream().collect(Collectors.groupingBy(v -> v.projectName)).entrySet().stream() - .forEach(v -> { - if (v.getValue().size() > 1) { - v.getValue().stream().skip(1).forEach(Project::delete); - } - }); - } - private List getRepoUrls(Path miningFile) throws IOException { String repos = StringUtils.substringBetween(Files.readString(miningFile), "", ""); diff --git a/github-bot/src/main/java/io/github/martinwitt/laughing_train/services/QodanaService.java b/github-bot/src/main/java/io/github/martinwitt/laughing_train/services/QodanaService.java index d44869e00..ff1cf1058 100644 --- a/github-bot/src/main/java/io/github/martinwitt/laughing_train/services/QodanaService.java +++ b/github-bot/src/main/java/io/github/martinwitt/laughing_train/services/QodanaService.java @@ -121,10 +121,7 @@ private QodanaResult invokeQodana(AnalyzerRequest.WithProject project, ProjectCo return threadPoolManager .getService() .submit(() -> new QodanaResult.Success( - runQodana( - project.project().folder().toPath(), - projectConfig.getSourceFolder(), - project.project().name()), + runQodana(project.project().folder().toPath(), projectConfig.getSourceFolder()), project.project())) .get(); } catch (Exception e) { @@ -132,10 +129,9 @@ private QodanaResult invokeQodana(AnalyzerRequest.WithProject project, ProjectCo } } - private List runQodana(Path path, String sourceFolder, String projectName) { + private List runQodana(Path path, String sourceFolder) { QodanaAnalyzer analyzer = new QodanaAnalyzer.Builder() .withSourceFileRoot(sourceFolder) - .withCacheVolume("lauging-train.qodana-cache", projectName) .withResultFolder(path.toAbsolutePath().toString()) .build(); return analyzer.runQodana(path); From 281af86e7c4c2711126745d206fdd893d330325b Mon Sep 17 00:00:00 2001 From: Martin Wittlinger Date: Thu, 27 Oct 2022 17:41:36 +0200 Subject: [PATCH 11/13] update cosign --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 6a71ba2f6..cced14973 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -44,7 +44,7 @@ jobs: if: github.event_name != 'pull_request' uses: sigstore/cosign-installer@7bca8b41164994a7dc93749d266e2f1db492f8a2 with: - cosign-release: 'v1.9.0' + cosign-release: 'v1.13.1' # Build Quarkus - name: Build run: cd ./github-bot/ && gradle assemble From 755182a1bf0138472b599a7cc127a4f9476fc910 Mon Sep 17 00:00:00 2001 From: Martin Wittlinger Date: Thu, 27 Oct 2022 17:55:58 +0200 Subject: [PATCH 12/13] test assertJ --- .../github/martinwitt/laughing_train/mining/PeriodicMiner.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/PeriodicMiner.java b/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/PeriodicMiner.java index 86826b048..4dea83323 100644 --- a/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/PeriodicMiner.java +++ b/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/PeriodicMiner.java @@ -49,7 +49,7 @@ public class PeriodicMiner { void mineRepos() { // for (Project project : Project.findAll().list()) { for (Project project : Project.findAll().stream() - .filter(v -> v.getProjectName().contains("guava")) + .filter(v -> v.getProjectUrl().toLowerCase().contains("assert")) .toList()) { eventBus.request( ServiceAdresses.PROJECT_REQUEST, From e1bea424b664e6270388dca3f24a6c315c7b3140 Mon Sep 17 00:00:00 2001 From: Martin Wittlinger Date: Thu, 27 Oct 2022 19:15:13 +0200 Subject: [PATCH 13/13] up --- frontend/package-lock.json | 39 ++++-------------- frontend/src/ProjectData.tsx | 16 ++++++++ frontend/src/data/ProjectConfig.tsx | 4 ++ frontend/src/index.tsx | 5 +++ frontend/src/pages/ProjectConfigView.tsx | 41 +++++++++++++++++++ frontend/src/pages/Resultview.tsx | 11 ++++- .../laughing_train/mining/PeriodicMiner.java | 5 +-- 7 files changed, 85 insertions(+), 36 deletions(-) create mode 100644 frontend/src/data/ProjectConfig.tsx create mode 100644 frontend/src/pages/ProjectConfigView.tsx diff --git a/frontend/package-lock.json b/frontend/package-lock.json index de6b1d330..b9a7efb91 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -17593,25 +17593,14 @@ } }, "node_modules/recursive-readdir": { - "version": "2.2.2", - "resolved": "https://registry.npmjs.org/recursive-readdir/-/recursive-readdir-2.2.2.tgz", - "integrity": "sha512-nRCcW9Sj7NuZwa2XvH9co8NPeXUBhZP7CRKJtU+cS6PW9FpCIFoI5ib0NT1ZrbNuPoRy0ylyCaUL8Gih4LSyFg==", - "dependencies": { - "minimatch": "3.0.4" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/recursive-readdir/node_modules/minimatch": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/recursive-readdir/-/recursive-readdir-2.2.3.tgz", + "integrity": "sha512-8HrF5ZsXk5FAH9dgsx3BlUer73nIhuj+9OrQwEbLTPOBzGkL1lsFCR01am+v+0m2Cmbs1nP12hLDl5FA7EszKA==", "dependencies": { - "brace-expansion": "^1.1.7" + "minimatch": "^3.0.5" }, "engines": { - "node": "*" + "node": ">=6.0.0" } }, "node_modules/redent": { @@ -33015,21 +33004,11 @@ } }, "recursive-readdir": { - "version": "2.2.2", - "resolved": "https://registry.npmjs.org/recursive-readdir/-/recursive-readdir-2.2.2.tgz", - "integrity": "sha512-nRCcW9Sj7NuZwa2XvH9co8NPeXUBhZP7CRKJtU+cS6PW9FpCIFoI5ib0NT1ZrbNuPoRy0ylyCaUL8Gih4LSyFg==", + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/recursive-readdir/-/recursive-readdir-2.2.3.tgz", + "integrity": "sha512-8HrF5ZsXk5FAH9dgsx3BlUer73nIhuj+9OrQwEbLTPOBzGkL1lsFCR01am+v+0m2Cmbs1nP12hLDl5FA7EszKA==", "requires": { - "minimatch": "3.0.4" - }, - "dependencies": { - "minimatch": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", - "requires": { - "brace-expansion": "^1.1.7" - } - } + "minimatch": "^3.0.5" } }, "redent": { diff --git a/frontend/src/ProjectData.tsx b/frontend/src/ProjectData.tsx index e698f4155..8fae7bfaa 100644 --- a/frontend/src/ProjectData.tsx +++ b/frontend/src/ProjectData.tsx @@ -72,3 +72,19 @@ export function filterDuplicateBadSmells(params: BadSmell[]) { return filtered; } +export const fetchProjectConfigQuery = gql` + query getProjectConfig($projectUrl: String!) { + getProjectConfig(projectUrl: $projectUrl) { + projectUrl + sourceFolder + } +} +`; +export const addProjectConfigQuery = gql` + mutation addProjectConfig($projectConfig: ProjectConfig!) { + addProjectConfig(projectConfig: $projectConfig) { + projectUrl + sourceFolder + } +} +`; \ No newline at end of file diff --git a/frontend/src/data/ProjectConfig.tsx b/frontend/src/data/ProjectConfig.tsx new file mode 100644 index 000000000..a90dc7297 --- /dev/null +++ b/frontend/src/data/ProjectConfig.tsx @@ -0,0 +1,4 @@ +export type ProjectConfig = { + projectUrl: string, + sourceFolder: string +} \ No newline at end of file diff --git a/frontend/src/index.tsx b/frontend/src/index.tsx index 84c16e6d0..19d112410 100644 --- a/frontend/src/index.tsx +++ b/frontend/src/index.tsx @@ -12,6 +12,7 @@ import { ThemeProvider, createTheme, ThemeOptions } from '@mui/material/styles'; import { CssBaseline } from '@mui/material'; import { AddProjectView } from './pages/AddProjectView'; import { RefactorView } from './pages/RefactorView'; +import { ProjectConfigview } from './pages/ProjectConfigView'; const root = ReactDOM.createRoot( document.getElementById('root') as HTMLElement ); @@ -30,6 +31,10 @@ const router = createBrowserRouter([ path: "/mutation/refactor/:name/:hash", element: , }, + { + path: "/mutation/projectconfig/:projectUrl", + element: , + }, { path: "/resultview", element: , diff --git a/frontend/src/pages/ProjectConfigView.tsx b/frontend/src/pages/ProjectConfigView.tsx new file mode 100644 index 000000000..6eecd43d6 --- /dev/null +++ b/frontend/src/pages/ProjectConfigView.tsx @@ -0,0 +1,41 @@ +import { useQuery } from '@apollo/client'; +import { Alert, Box, Button, TextField, Typography } from '@mui/material'; +import React, { useState } from 'react'; +import { useParams } from 'react-router'; +import Headline from '../component/Headline'; +import { fetchProjectConfigQuery } from '../ProjectData'; + +export function ProjectConfigview() { + const { projectUrl } = useParams(); + const encodedProjectUrl = toBase64(projectUrl); + const [sourceFolder, setSourceFolder] = useState("") + const { loading, error} = useQuery(fetchProjectConfigQuery, { + variables: { projectUrl: encodedProjectUrl }, + onCompleted: (data) => { + setSourceFolder(data.getProjectConfig.sourceFolder) + } + }); + return ( +
+ + +
+ Project Config +
+ {loading &&

Loading...

} + {error && Can not fetch data. Are you logged in?} + setSourceFolder(e.target.value)} /> + + +
+ ); +} +function toBase64(str: string | undefined): string { + if (undefined === str) { + return ""; + } + return atob(str); +} \ No newline at end of file diff --git a/frontend/src/pages/Resultview.tsx b/frontend/src/pages/Resultview.tsx index b0af5e4e8..d4a4f3104 100644 --- a/frontend/src/pages/Resultview.tsx +++ b/frontend/src/pages/Resultview.tsx @@ -9,7 +9,6 @@ import { Project } from "../data/Project"; import { useQuery } from "@apollo/client"; import BadSmellList from "../component/BadSmellList"; - function Resultview() { let params = useParams(); @@ -40,7 +39,9 @@ function Resultview() {
- + + + ); } @@ -62,4 +63,10 @@ function addHashIfPresent(hash: string | undefined) : string { return hash; } return ""; +} +function toBase64(str: string) { + return btoa(str); +} +function generateProjectConfigLink(project: Project) { + return "/mutation/projectconfig/" + toBase64(project.projectUrl); } \ No newline at end of file diff --git a/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/PeriodicMiner.java b/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/PeriodicMiner.java index 4dea83323..056c8ade3 100644 --- a/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/PeriodicMiner.java +++ b/github-bot/src/main/java/io/github/martinwitt/laughing_train/mining/PeriodicMiner.java @@ -47,10 +47,7 @@ public class PeriodicMiner { @Scheduled(every = "4h", delay = 3, delayUnit = TimeUnit.MINUTES) void mineRepos() { - // for (Project project : Project.findAll().list()) { - for (Project project : Project.findAll().stream() - .filter(v -> v.getProjectUrl().toLowerCase().contains("assert")) - .toList()) { + for (Project project : Project.findAll().list()) { eventBus.request( ServiceAdresses.PROJECT_REQUEST, new ProjectRequest.WithUrl(project.getProjectUrl()),