#6 - Implementing stars-up bot (#18)

* #6 - `Stars-up` bot: work in progress
* #6 - Changing puzzles format according to pdd
* #6 - Integration tests for StarsUp bot
* #6 - Another more integration tests for StarsUp bot
* #6 - StarsUp big methods decomposition

README.md

@@ -20,6 +20,11 @@ Bots list:
 - stars-up: `The repo gained 120 stars in 7 days`
 - forks-up: `The repo was forked 40 times in 7 days`
      5% increase/decrease in both metrics
+     If the difference is smaller than 10 stars, the bot stays quiet.
+     From 0 to 200 stars bot will be sending 1 review per 10 stars increase.
+     From 200 to 220 stars bot will be sending 1 review per 11 stars increase.
+     From 220 to 240 stars bot will be sending 1 review per 12 stars increase.
+     And so on.
 
 
 ### Ideas backlog

build.gradle

@@ -47,6 +47,7 @@ dependencies {
 
     implementation 'de.siegmar:logback-gelf:2.0.1'
     implementation 'org.cactoos:cactoos:0.43'
+    implementation 'org.javatuples:javatuples:1.2'
 
     implementation 'org.springframework.cloud:spring-cloud-starter-openfeign:2.1.1.RELEASE'
     implementation 'io.github.openfeign:feign-jackson:9.3.1'
@@ -65,6 +66,7 @@ dependencies {
     testImplementation 'com.github.javafaker:javafaker:1.0.2'
     testImplementation 'com.github.tomakehurst:wiremock:2.26.3'
     testImplementation 'org.codehaus.groovy:groovy-all:3.0.2'
+    testImplementation 'org.junit.jupiter:junit-jupiter-params:5.6.2'
 }
 
 springBoot {
@@ -84,6 +86,7 @@ lombok {
     version = '1.18.10'
     config['lombok.accessors.chain'] = 'true'
     config['lombok.equalsAndHashCode.callSuper'] = 'skip'
+    config['lombok.toString.callSuper'] = 'call'
 }
 
 sonarqube {

lombok.config

@@ -3,3 +3,4 @@ config.stopBubbling = true
 lombok.accessors.chain = true
 lombok.addLombokGeneratedAnnotation = true
 lombok.equalsAndHashCode.callSuper = skip
+lombok.toString.callSuper = call

src/main/java/dev/iakunin/codexiabot/bot/Bot.java

@@ -4,5 +4,7 @@ public interface Bot {
     enum Type {
         FOUND_ON_HACKERNEWS,
         FOUND_ON_REDDIT,
+        STARS_UP,
+        FORKS_UP,
     }
 }

src/main/java/dev/iakunin/codexiabot/bot/FoundOnHackernews.java

@@ -51,18 +51,12 @@ public void run() {
                 githubRepo -> {
                     final Set<GithubRepoSource> allRepoSources = this.githubModule.findAllRepoSources(githubRepo);
 
-                    final CodexiaProject codexiaProject = allRepoSources.stream()
-                        .filter(source -> source.getSource() == GithubModule.Source.CODEXIA)
-                        .findFirst()
-                        .flatMap(
-                            source -> this.codexiaModule.findByExternalId(Integer.valueOf(source.getExternalId()))
-                        )
+                    final CodexiaProject codexiaProject = this.codexiaModule
+                        .findCodexiaProject(githubRepo)
                         .orElseThrow(
                             () -> new RuntimeException(
                                 String.format(
-                                    "Unable to find source with type='%s' for githubRepoId='%s' " +
-                                    "or codexiaProject by externalId",
-                                    GithubModule.Source.CODEXIA.name(),
+                                    "Unable to find CodexiaProject for githubRepoId='%s'",
                                     githubRepo.getId()
                                 )
                             )
@@ -116,6 +110,7 @@ public void run() {
     }
 
     @Value
+    // @todo #6 get rid of FoundOnHackernews.TmpDto using `org.javatuples.Pair`
     private static final class TmpDto {
         private CodexiaProject codexiaProject;
         private GithubRepoSource hackernewsSource;

src/main/java/dev/iakunin/codexiabot/bot/FoundOnReddit.java

@@ -35,18 +35,12 @@ public void run() {
                 githubRepo -> {
                     final Set<GithubRepoSource> allRepoSources = this.githubModule.findAllRepoSources(githubRepo);
 
-                    final CodexiaProject codexiaProject = allRepoSources.stream()
-                        .filter(source -> source.getSource() == GithubModule.Source.CODEXIA)
-                        .findFirst()
-                        .flatMap(
-                            source -> this.codexiaModule.findByExternalId(Integer.valueOf(source.getExternalId()))
-                        )
+                    final CodexiaProject codexiaProject = this.codexiaModule
+                        .findCodexiaProject(githubRepo)
                         .orElseThrow(
                             () -> new RuntimeException(
                                 String.format(
-                                    "Unable to find source with type='%s' for githubRepoId='%s' " +
-                                    "or codexiaProject by externalId",
-                                    GithubModule.Source.CODEXIA.name(),
+                                    "Unable to find CodexiaProject for githubRepoId='%s'",
                                     githubRepo.getId()
                                 )
                             )
@@ -100,6 +94,7 @@ public void run() {
     }
 
     @Value
+    // @todo #6 get rid of FoundOnReddit.TmpDto using `org.javatuples.Pair`
     private static final class TmpDto {
         private CodexiaProject codexiaProject;
         private GithubRepoSource redditSource;

src/main/java/dev/iakunin/codexiabot/bot/StarsUp.java

@@ -0,0 +1,138 @@
+package dev.iakunin.codexiabot.bot;
+
+import dev.iakunin.codexiabot.bot.entity.StarsUpResult;
+import dev.iakunin.codexiabot.bot.repository.StarsUpResultRepository;
+import dev.iakunin.codexiabot.codexia.CodexiaModule;
+import dev.iakunin.codexiabot.codexia.entity.CodexiaProject;
+import dev.iakunin.codexiabot.codexia.entity.CodexiaReview;
+import dev.iakunin.codexiabot.github.GithubModule;
+import dev.iakunin.codexiabot.github.entity.GithubRepo;
+import dev.iakunin.codexiabot.github.entity.GithubRepoStat;
+import dev.iakunin.codexiabot.github.entity.GithubRepoStat.GithubApi;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+import java.util.Deque;
+import java.util.stream.Collectors;
+import lombok.AllArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.javatuples.Pair;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+import org.springframework.transaction.annotation.Transactional;
+
+@Component
+@Slf4j
+@AllArgsConstructor(onConstructor_={@Autowired})
+public class StarsUp {
+
+    private static final Bot.Type BOT_TYPE = Bot.Type.STARS_UP;
+
+    private final GithubModule githubModule;
+
+    private final CodexiaModule codexiaModule;
+
+    private final StarsUpResultRepository starsUpResultRepository;
+
+    @Scheduled(cron="${app.cron.bot.stars-up:-}")
+    public void run() {
+        log.info("Running {}", this.getClass().getName());
+
+        this.githubModule.findAllInCodexia()
+            .stream()
+            .map(
+                repo -> Pair.with(repo, this.getLastProcessedStatId(repo))
+            )
+            .map(
+                pair -> this.githubModule.findAllGithubApiStat(
+                    pair.getValue0(),
+                    pair.getValue1()
+                )
+            )
+            .filter(statList -> statList.size() >= 2)
+            .filter(
+                statList -> this.shouldReviewBeSubmitted(
+                    (GithubApi) statList.getFirst().getStat(),
+                    (GithubApi) statList.getLast().getStat()
+                )
+            )
+            .forEach(this::processStatList);
+
+        log.info("Exiting from {}", this.getClass().getName());
+    }
+
+    private Long getLastProcessedStatId(GithubRepo repo) {
+        return this.starsUpResultRepository
+            .findFirstByGithubRepoOrderByIdDesc(repo)
+            .map(
+                starsUpResult -> starsUpResult.getGithubRepoStat().getId()
+            )
+            .orElse(0L);
+    }
+
+    // @todo #6 add test case with transaction rollback
+    @Transactional
+    protected void processStatList(Deque<GithubRepoStat> deque) {
+        final CodexiaReview review = this.createReview(deque.getFirst(), deque.getLast());
+
+        this.starsUpResultRepository.save(
+            new StarsUpResult()
+                .setGithubRepo(deque.getLast().getGithubRepo())
+                .setGithubRepoStat(deque.getLast())
+        );
+        this.codexiaModule.sendReview(review);
+        this.codexiaModule.sendMeta(
+            review.getCodexiaProject(),
+            "stars-count",
+            this.codexiaModule
+                .findAllReviews(review.getCodexiaProject(), review.getAuthor())
+                .stream()
+                .map(CodexiaReview::getReason)
+                .collect(Collectors.joining(","))
+        );
+    }
+
+    private CodexiaReview createReview(GithubRepoStat first, GithubRepoStat last) {
+        final GithubApi firstStat = (GithubApi) first.getStat();
+        final GithubApi lastStat = (GithubApi) last.getStat();
+
+        return new CodexiaReview()
+            .setText(
+                String.format(
+                    "The repo gained %d stars since %s (was: %d stars, now: %d stars). " +
+                    "See the stars history [here](https://star-history.t9t.io/#%s).",
+                    lastStat.getStars() - firstStat.getStars(),
+                    ZonedDateTime.of(first.getCreatedAt(), ZoneOffset.UTC).toString(),
+                    firstStat.getStars(),
+                    lastStat.getStars(),
+                    first.getGithubRepo().getFullName()
+                )
+            )
+            .setAuthor(BOT_TYPE.name())
+            .setReason(String.valueOf(lastStat.getStars()))
+            .setCodexiaProject(this.getCodexiaProject(last));
+    }
+
+    private CodexiaProject getCodexiaProject(GithubRepoStat last) {
+        return this.codexiaModule
+            .findCodexiaProject(last.getGithubRepo())
+            .orElseThrow(
+                () -> new RuntimeException(
+                    String.format(
+                        "Unable to find CodexiaProject for githubRepoId='%s'",
+                        last.getGithubRepo().getId()
+                    )
+                )
+            );
+    }
+
+    private boolean shouldReviewBeSubmitted(GithubApi first, GithubApi last) {
+        final int increase = last.getStars() - first.getStars();
+
+        if (increase < 10) {
+            return false;
+        }
+
+        return increase >= (first.getStars() * 0.05);
+    }
+}

src/main/java/dev/iakunin/codexiabot/bot/entity/StarsUpResult.java

@@ -0,0 +1,21 @@
+package dev.iakunin.codexiabot.bot.entity;
+
+import dev.iakunin.codexiabot.common.entity.AbstractEntity;
+import dev.iakunin.codexiabot.github.entity.GithubRepo;
+import dev.iakunin.codexiabot.github.entity.GithubRepoStat;
+import javax.persistence.Entity;
+import javax.persistence.ManyToOne;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+
+@Entity
+@Data
+@EqualsAndHashCode(callSuper = true, onlyExplicitlyIncluded = true)
+public final class StarsUpResult extends AbstractEntity {
+
+    @ManyToOne
+    private GithubRepo githubRepo;
+
+    @ManyToOne
+    private GithubRepoStat githubRepoStat;
+}

src/main/java/dev/iakunin/codexiabot/bot/repository/StarsUpResultRepository.java

@@ -0,0 +1,11 @@
+package dev.iakunin.codexiabot.bot.repository;
+
+import dev.iakunin.codexiabot.bot.entity.StarsUpResult;
+import dev.iakunin.codexiabot.github.entity.GithubRepo;
+import java.util.Optional;
+import org.springframework.data.jpa.repository.JpaRepository;
+
+public interface StarsUpResultRepository extends JpaRepository<StarsUpResult, Long> {
+
+    Optional<StarsUpResult> findFirstByGithubRepoOrderByIdDesc(GithubRepo repo);
+}

src/main/java/dev/iakunin/codexiabot/codexia/CodexiaModule.java

@@ -2,16 +2,23 @@
 
 import dev.iakunin.codexiabot.codexia.entity.CodexiaProject;
 import dev.iakunin.codexiabot.codexia.entity.CodexiaReview;
+import dev.iakunin.codexiabot.github.entity.GithubRepo;
 import java.util.List;
 import java.util.Optional;
 
 public interface CodexiaModule {
 
+    // @todo #6 split codexiaModule.sendReview() to codexiaModule.createReview() and codexiaModule.sendReview()
+    //  - codexiaModule.createReview() - just a saving to DB.
+    //  - codexiaModule.sendReview() - real sending review to Codexia api.
+    //  - codexiaModule.sendReview() must be used ONLY inside a cron-job.
+    //  - In other code there should be only a codexiaModule.createReview() calls.
+    //  - All the codexiaModule.createReview() calls MUST be inside a transaction/
     void sendReview(CodexiaReview review);
 
     void sendMeta(CodexiaProject codexiaProject, String metaKey, String metaValue);
 
-    Optional<CodexiaProject> findByExternalId(Integer externalId);
+    Optional<CodexiaProject> findCodexiaProject(GithubRepo repo);
 
     boolean isReviewExist(CodexiaProject codexiaProject, String author, String reason);
 

src/main/java/dev/iakunin/codexiabot/codexia/CodexiaModuleImpl.java

@@ -7,6 +7,8 @@
 import dev.iakunin.codexiabot.codexia.repository.CodexiaReviewNotificationRepository;
 import dev.iakunin.codexiabot.codexia.repository.CodexiaReviewRepository;
 import dev.iakunin.codexiabot.codexia.sdk.CodexiaClient;
+import dev.iakunin.codexiabot.github.GithubModule;
+import dev.iakunin.codexiabot.github.entity.GithubRepo;
 import feign.FeignException;
 import java.util.List;
 import java.util.Optional;
@@ -31,6 +33,8 @@ public final class CodexiaModuleImpl implements CodexiaModule {
 
     private final CodexiaClient codexiaClient;
 
+    private final GithubModule githubModule;
+
     @Override
     public void sendReview(CodexiaReview review) {
         log.info("Got a review: {}", review);
@@ -64,7 +68,7 @@ public void sendReview(CodexiaReview review) {
                             : CodexiaReviewNotification.Status.ERROR
                     )
                     .setResponseCode(e.status())
-                    .setResponse(e.contentUTF8())
+                    .setResponse(e.content() != null ? e.contentUTF8() : "")
             );
             return;
         }
@@ -95,8 +99,17 @@ public void sendMeta(CodexiaProject codexiaProject, String metaKey, String metaV
     }
 
     @Override
-    public Optional<CodexiaProject> findByExternalId(Integer externalId) {
-        return this.codexiaProjectRepository.findByExternalId(externalId);
+    public Optional<CodexiaProject> findCodexiaProject(GithubRepo repo) {
+        return this.githubModule
+            .findAllRepoSources(repo)
+            .stream()
+            .filter(source -> source.getSource() == GithubModule.Source.CODEXIA)
+            .findFirst()
+            .flatMap(
+                source -> this.codexiaProjectRepository.findByExternalId(
+                    Integer.valueOf(source.getExternalId())
+                )
+            );
     }
 
     @Override
@@ -106,6 +119,6 @@ public boolean isReviewExist(CodexiaProject codexiaProject, String author, Strin
 
     @Override
     public List<CodexiaReview> findAllReviews(CodexiaProject codexiaProject, String author) {
-        return this.codexiaReviewRepository.findAllByCodexiaProjectAndAuthor(codexiaProject, author);
+        return this.codexiaReviewRepository.findAllByCodexiaProjectAndAuthorOrderByIdAsc(codexiaProject, author);
     }
 }

src/main/java/dev/iakunin/codexiabot/codexia/repository/CodexiaReviewRepository.java

@@ -9,5 +9,5 @@ public interface CodexiaReviewRepository extends JpaRepository<CodexiaReview, Lo
 
     boolean existsByCodexiaProjectAndAuthorAndReason(CodexiaProject codexiaProject, String author, String reason);
 
-    List<CodexiaReview> findAllByCodexiaProjectAndAuthor(CodexiaProject codexiaProject, String author);
+    List<CodexiaReview> findAllByCodexiaProjectAndAuthorOrderByIdAsc(CodexiaProject codexiaProject, String author);
 }

src/main/java/dev/iakunin/codexiabot/codexia/sdk/CodexiaClient.java

@@ -70,7 +70,9 @@ class Project {
         private Integer authorId;
         private String deleted;
 
+        // @todo #6 write tests for different timezones
         @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd HH:mm:ss Z")
+        // @todo #6 replace `Date` with `ZonedDateTime`
         private Date created;
     }
 }

src/main/java/dev/iakunin/codexiabot/common/entity/AbstractEntity.java

@@ -33,6 +33,14 @@ public abstract class AbstractEntity {
     @Column(updatable = false)
     @CreationTimestamp
     private LocalDateTime createdAt;
+    // @todo #6 choose better created_at postgres-format (with or without TZ).
+    //  Also think of ZonedDateTime in createdAt field.
+    //  ZonedDateTime + timestamptz seems the best way.
+    //  - Test changing of postgres server timezone.
+    //  - Test changing of hibernate.time_zone property.
+    //  - Test changing of JVM timezone.
+    //  In all of these cases there MUST be valid timestamptz in postgres.
+    //  In all of these cases there MUST be valid ZonedDateTime in code.
 
     @UpdateTimestamp
     private LocalDateTime updatedAt;