Development: Optimize local git repository cleanup

src/main/java/de/tum/cit/aet/artemis/programming/repository/ProgrammingExerciseRepository.java

@@ -226,26 +226,6 @@ default ProgrammingExercise findOneByProjectKeyOrThrow(String projectKey, boolea
             """)
     List<ProgrammingExercise> findAllByRecentExamEndDate(@Param("endDate1") ZonedDateTime endDate1, @Param("endDate2") ZonedDateTime endDate2);
 
-    @Query("""
-            SELECT DISTINCT pe
-            FROM ProgrammingExercise pe
-                LEFT JOIN FETCH pe.studentParticipations
-            WHERE pe.dueDate IS NOT NULL
-                AND :endDate1 <= pe.dueDate
-                AND pe.dueDate <= :endDate2
-            """)
-    List<ProgrammingExercise> findAllWithStudentParticipationByRecentDueDate(@Param("endDate1") ZonedDateTime endDate1, @Param("endDate2") ZonedDateTime endDate2);
-
-    @Query("""
-            SELECT DISTINCT pe
-            FROM ProgrammingExercise pe
-                LEFT JOIN FETCH pe.studentParticipations
-            WHERE pe.exerciseGroup IS NOT NULL
-                AND :endDate1 <= pe.exerciseGroup.exam.endDate
-                AND pe.exerciseGroup.exam.endDate <= :endDate2
-            """)
-    List<ProgrammingExercise> findAllWithStudentParticipationByRecentExamEndDate(@Param("endDate1") ZonedDateTime endDate1, @Param("endDate2") ZonedDateTime endDate2);
-
     @Query("""
             SELECT DISTINCT pe
             FROM ProgrammingExercise pe

src/main/java/de/tum/cit/aet/artemis/programming/repository/ProgrammingExerciseStudentParticipationRepository.java

@@ -9,6 +9,8 @@
 import java.util.Optional;
 
 import org.springframework.context.annotation.Profile;
+import org.springframework.data.domain.Page;
+import org.springframework.data.domain.Pageable;
 import org.springframework.data.jpa.repository.EntityGraph;
 import org.springframework.data.jpa.repository.Modifying;
 import org.springframework.data.jpa.repository.Query;
@@ -164,6 +166,19 @@ List<ProgrammingExerciseStudentParticipation> findWithSubmissionsByExerciseIdAnd
     Optional<ProgrammingExerciseStudentParticipation> findWithSubmissionsByExerciseIdAndStudentLoginAndTestRun(@Param("exerciseId") long exerciseId,
             @Param("username") String username, @Param("testRun") boolean testRun);
 
+    @Query("""
+            SELECT participation.repositoryUri
+            FROM ProgrammingExerciseStudentParticipation participation
+                JOIN TREAT (participation.exercise AS ProgrammingExercise) pe
+            WHERE participation.repositoryUri IS NOT NULL
+                AND (
+                    (pe.dueDate IS NOT NULL AND :latestDate <= pe.dueDate AND pe.dueDate <= :earliestDate)
+                    OR (pe.exerciseGroup IS NOT NULL AND :latestDate <= pe.exerciseGroup.exam.endDate AND pe.exerciseGroup.exam.endDate <= :earliestDate)
+                )
+            """)
+    Page<String> findRepositoryUrisForGitCleanupByRecentDueDateOrRecentExamEndDate(@Param("earliestDate") ZonedDateTime earliestDate, @Param("latestDate") ZonedDateTime latestDate,
+            Pageable pageable);
+
     @Query("""
             SELECT participation
             FROM ProgrammingExerciseStudentParticipation participation

src/main/java/de/tum/cit/aet/artemis/programming/service/AutomaticProgrammingExerciseCleanupService.java

@@ -3,19 +3,23 @@
 import static de.tum.cit.aet.artemis.core.config.Constants.PROFILE_SCHEDULING;
 import static java.time.ZonedDateTime.now;
 
+import java.net.URISyntaxException;
 import java.time.ZonedDateTime;
 import java.time.temporal.ChronoUnit;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import org.hibernate.Hibernate;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Profile;
+import org.springframework.data.domain.Page;
+import org.springframework.data.domain.Pageable;
 import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Service;
 
@@ -25,6 +29,7 @@
 import de.tum.cit.aet.artemis.exercise.service.ParticipationService;
 import de.tum.cit.aet.artemis.programming.domain.ProgrammingExercise;
 import de.tum.cit.aet.artemis.programming.domain.ProgrammingExerciseStudentParticipation;
+import de.tum.cit.aet.artemis.programming.domain.VcsRepositoryUri;
 import de.tum.cit.aet.artemis.programming.repository.ProgrammingExerciseRepository;
 import de.tum.cit.aet.artemis.programming.repository.ProgrammingExerciseStudentParticipationRepository;
 
@@ -44,6 +49,8 @@ public class AutomaticProgrammingExerciseCleanupService {
 
     private final GitService gitService;
 
+    private static final int STUDENT_PARTICIPATION_CLEANUP_BATCH_SIZE = 500;
+
     @Value("${artemis.external-system-request.batch-size}")
     private int externalSystemRequestBatchSize;
 
@@ -82,49 +89,68 @@ public void cleanup() {
             log.error("Exception occurred during cleanupBuildPlansOnContinuousIntegrationServer", ex);
         }
         try {
-            cleanupGitRepositoriesOnArtemisServer();
+            cleanupGitWorkingCopiesOnArtemisServer();
         }
         catch (Exception ex) {
-            log.error("Exception occurred during cleanupGitRepositoriesOnArtemisServer", ex);
+            log.error("Exception occurred during cleanupGitWorkingCopiesOnArtemisServer", ex);
         }
     }
 
     /**
      * cleans up old local git repositories on the Artemis server
      */
-    public void cleanupGitRepositoriesOnArtemisServer() {
+    public void cleanupGitWorkingCopiesOnArtemisServer() {
         SecurityUtils.setAuthorizationObject();
         log.info("Cleanup git repositories on Artemis server");
         // we are specifically interested in exercises older than 8 weeks
-        var endDate2 = ZonedDateTime.now().minusWeeks(8).truncatedTo(ChronoUnit.DAYS);
+        var earliestDate = ZonedDateTime.now().minusWeeks(8).truncatedTo(ChronoUnit.DAYS);
         // NOTE: for now we would like to cover more cases to also cleanup older repositories
-        var endDate1 = endDate2.minusYears(1).truncatedTo(ChronoUnit.DAYS);
-
-        // Cleanup all student repos in the REPOS folder (based on the student participations) 8 weeks after the exercise due date
-        log.info("Search for exercises with due date from {} until {}", endDate1, endDate2);
-        var programmingExercises = programmingExerciseRepository.findAllWithStudentParticipationByRecentDueDate(endDate1, endDate2);
-        programmingExercises.addAll(programmingExerciseRepository.findAllWithStudentParticipationByRecentExamEndDate(endDate1, endDate2));
-        log.info("Found {} programming exercises {} to clean {} local student repositories", programmingExercises.size(),
-                programmingExercises.stream().map(ProgrammingExercise::getProjectKey).collect(Collectors.joining(", ")),
-                programmingExercises.stream().mapToLong(programmingExercise -> programmingExercise.getStudentParticipations().size()).sum());
-        for (var programmingExercise : programmingExercises) {
-            for (var studentParticipation : programmingExercise.getStudentParticipations()) {
-                var programmingExerciseParticipation = (ProgrammingExerciseStudentParticipation) studentParticipation;
-                gitService.deleteLocalRepository(programmingExerciseParticipation.getVcsRepositoryUri());
-            }
-        }
+        var latestDate = earliestDate.minusYears(1).truncatedTo(ChronoUnit.DAYS);
+
+        // Cleanup all student repos in the REPOS folder (based on the student participations) 8 weeks after the exercise due date or exam end date
+        cleanStudentParticipationsRepositories(earliestDate, latestDate);
 
         // Cleanup template, tests and solution repos in the REPOS folder 8 weeks after the course or exam is over
-        log.info("Search for exercises with course or exam date from {} until {}", endDate1, endDate2);
-        programmingExercises = programmingExerciseRepository.findAllByRecentCourseEndDate(endDate1, endDate2);
-        programmingExercises.addAll(programmingExerciseRepository.findAllByRecentExamEndDate(endDate1, endDate2));
+        log.info("Search for exercises with course or exam date from {} until {}", latestDate, earliestDate);
+        var programmingExercises = programmingExerciseRepository.findAllByRecentCourseEndDate(latestDate, earliestDate);
+        programmingExercises.addAll(programmingExerciseRepository.findAllByRecentExamEndDate(latestDate, earliestDate));
         log.info("Found {} programming exercise to clean local template, test and solution: {}", programmingExercises.size(),
                 programmingExercises.stream().map(ProgrammingExercise::getProjectKey).collect(Collectors.joining(", ")));
-        for (var programmingExercise : programmingExercises) {
-            gitService.deleteLocalRepository(programmingExercise.getVcsTemplateRepositoryUri());
-            gitService.deleteLocalRepository(programmingExercise.getVcsSolutionRepositoryUri());
-            gitService.deleteLocalRepository(programmingExercise.getVcsTestRepositoryUri());
-            gitService.deleteLocalProgrammingExerciseReposFolder(programmingExercise);
+        if (!programmingExercises.isEmpty()) {
+            for (var programmingExercise : programmingExercises) {
+                gitService.deleteLocalRepository(programmingExercise.getVcsTemplateRepositoryUri());
+                gitService.deleteLocalRepository(programmingExercise.getVcsSolutionRepositoryUri());
+                gitService.deleteLocalRepository(programmingExercise.getVcsTestRepositoryUri());
+                gitService.deleteLocalProgrammingExerciseReposFolder(programmingExercise);
+            }
+            log.info("Finished cleaning local template, test and solution repositories");
+        }
+    }
+
+    private void cleanStudentParticipationsRepositories(ZonedDateTime earliestDate, ZonedDateTime latestDate) {
+        log.info("Search for exercises with due date from {} until {}", latestDate, earliestDate);
+        // Get all relevant participation ids
+        Pageable pageable = Pageable.ofSize(STUDENT_PARTICIPATION_CLEANUP_BATCH_SIZE);
+        Page<String> uriBatch = programmingExerciseStudentParticipationRepository.findRepositoryUrisForGitCleanupByRecentDueDateOrRecentExamEndDate(earliestDate, latestDate,
+                pageable);
+        log.info("Found {} student participations to clean local student repositories in {} batches.", uriBatch.getTotalElements(), uriBatch.getTotalPages());
+        if (uriBatch.getTotalElements() > 0) {
+            var ignored = Stream.iterate(uriBatch, Page::hasNext, page -> {
+                page.forEach(this::deleteLocalRepositoryByUriString);
+                return programmingExerciseStudentParticipationRepository.findRepositoryUrisForGitCleanupByRecentDueDateOrRecentExamEndDate(earliestDate, latestDate,
+                        page.nextPageable());
+            });
+            log.info("Finished cleaning local student repositories");
+        }
+    }
+
+    private void deleteLocalRepositoryByUriString(String uri) {
+        try {
+            VcsRepositoryUri vcsRepositoryUrl = new VcsRepositoryUri(uri);
+            gitService.deleteLocalRepository(vcsRepositoryUrl);
+        }
+        catch (URISyntaxException e) {
+            log.error("Cannot create URI for repositoryUri: {} due to the following error: {}", uri, e.getMessage());
         }
     }
 

src/test/java/de/tum/cit/aet/artemis/exercise/programming/ProgrammingExerciseTestService.java

@@ -2383,7 +2383,7 @@ void automaticCleanupGitRepositories() {
         createProgrammingParticipationWithSubmissionAndResult(examExercise, "student3", 100D, ZonedDateTime.now().minusDays(2L), false);
         createProgrammingParticipationWithSubmissionAndResult(examExercise, "student4", 80D, ZonedDateTime.now().minusDays(6L), false);
 
-        automaticProgrammingExerciseCleanupService.cleanupGitRepositoriesOnArtemisServer();
+        automaticProgrammingExerciseCleanupService.cleanupGitWorkingCopiesOnArtemisServer();
         // Note: at the moment, we cannot easily assert something here, it might be possible to verify mocks on gitService, in case we could define it as SpyBean
     }