Development: Add test file for script to generate random results when…

… creating large courses locally (#9298)

build.gradle

@@ -71,7 +71,8 @@ spotless {
                         "**/src/main/resources/templates/**",
                         "/docker/**",
                         "checked-out-repos/**",
-                        "**/src/main/java/org/eclipse/**"
+                        "**/src/main/java/org/eclipse/**",
+                        "supporting_scripts/**"
                 )
             }
         }

supporting_scripts/course-scripts/quick-course-setup/README.md

@@ -166,6 +166,30 @@ The script will automatically perform all the necessary steps:
 5. Create a programming exercise or use an existing one. 
 6. Add participation and commit for each student.
 
+### Optional: Generating Different Results For All Created Students (Should only be done Locally!!)
+
+If you want to generate different results for all the students created by the script:
+
+1.	Run the following Script which will navigate to the [testFiles](../../../src/main/resources/templates/java/test/testFiles) folder and copy the [RandomizedTestCases](./testFiles-template/randomized/RandomizedTestCases.java) file into it.
+    It will delete the existing folders (behavior and structural) from the programming exercise’s test case template. The new test cases will randomly pass or fail, causing different results for each student.
+```shell
+python3 randomize_results_before.py
+```
+2.  Rebuild Artemis to apply the changes.
+3.	Run the main method in large_course_main.py. Now, all created students should have varying results in the programming exercise.
+```shell
+python3 large_course_main.py
+```
+4.  Make sure to revert these changes after running the script. The following script copies the original test case files from the [default](./testFiles-template/default) folder back into the [testFiles](../../../src/main/resources/templates/java/test/testFiles) folder and deletes the [RandomizedTestCases](./testFiles-template/randomized/RandomizedTestCases.java) file that was copied to [testFiles](../../../src/main/resources/templates/java/test/testFiles) in Step 1.
+    If you don't run this script after running the script in Step 1, you risk breaking the real template of the programming exercise if these changes are pushed and merged.
+```shell
+python3 randomize_results_after.py
+```
+
+### Optional: Using an Existing Programming Exercise (Can also be done on Test Server)
+Alternatively, you can use an existing programming exercise and push the [RandomizedTestCases](./testFiles-template/randomized/RandomizedTestCases.java) file to the test repository of the programming exercise. 
+Make sure to adjust the [config.ini](./config.ini) file to use the existing programming exercise with the corresponding exercise ID, allowing the script to push with the created students to this existing programming exercise.
+
 ### Optional: Deleting All Created Students
 
 If you want to delete all the students created by the script:

supporting_scripts/course-scripts/quick-course-setup/create_course.py

@@ -8,6 +8,7 @@
 from requests import Session
 from utils import login_as_admin
 from add_users_to_course import add_users_to_groups_of_course
+from randomize_results_after import run_cleanup
 
 # Load configuration
 config = configparser.ConfigParser()
@@ -89,6 +90,7 @@ def create_course(session: Session) -> requests.Response:
         logging.info(f"Created course {COURSE_NAME} with shortName {course_short_name} \n {response.json()}")
     elif response.status_code == 400:
         logging.info(f"Course with shortName {course_short_name} already exists. Please provide the course ID in the config file and set create_course to FALSE if you intend to add programming exercises to this course.")
+        run_cleanup()
         sys.exit(0)
     else:
         logging.error("Problem with the group 'students' and interacting with a test server? "

supporting_scripts/course-scripts/quick-course-setup/large_course_main.py

@@ -7,6 +7,7 @@
 from create_users import create_students, user_credentials
 from add_users_to_course import add_students_to_groups_of_course
 from manage_programming_exercise import create_programming_exercise, add_participation, commit, exercise_Ids
+from randomize_results_after import run_cleanup
 
 # Load configuration and constants
 config = configparser.ConfigParser()
@@ -68,5 +69,8 @@ def main() -> None:
             commit(user_session, participation_id, CLIENT_URL, COMMITS_PER_STUDENT)
             logging.info(f"Added commit for {username} in the programming exercise {exercise_Id} successfully")
 
+    # This is a measure in case developers forget to revert changes to programming exercise template
+    run_cleanup()
+
 if __name__ == "__main__":
     main()

supporting_scripts/course-scripts/quick-course-setup/manage_programming_exercise.py

@@ -2,6 +2,7 @@
 from logging_config import logging
 from typing import Dict, Any
 from requests import Session
+from randomize_results_after import run_cleanup
 
 exercise_Ids: list[int] = []
 
@@ -39,6 +40,7 @@ def create_programming_exercise(session: Session, course_id: int, server_url: st
             exercise_Ids.append(response.json().get('id'))
         elif response.status_code == 400:
             logging.info(f"Programming exercise with shortName {default_programming_exercise['shortName']} already exists. Please provide the exercise IDs in the config file and set create_exercises to FALSE.")
+            run_cleanup()
             sys.exit(0)
         else:
             raise Exception(f"Could not create programming exercise; Status code: {response.status_code}\nResponse content: {response.text}")

supporting_scripts/course-scripts/quick-course-setup/randomize_results_after.py

@@ -0,0 +1,31 @@
+import os
+import shutil
+from logging_config import logging
+
+test_files_folder: str = "../../../src/main/resources/templates/java/test/testFiles"
+default_folder: str = "./testFiles-template/default"
+random_test_case_dest: str = os.path.join(test_files_folder, "RandomizedTestCases.java")
+
+def delete_random_test_case() -> None:
+    if os.path.exists(random_test_case_dest):
+        os.remove(random_test_case_dest)
+        logging.info(f"Deleted file: {random_test_case_dest}")
+    else:
+        logging.info(f"RandomizedTestCases.java file not found at {random_test_case_dest}")
+
+def copy_default_folders() -> None:
+    for folder_name in os.listdir(default_folder):
+        src_folder_path: str = os.path.join(default_folder, folder_name)
+        dest_folder_path: str = os.path.join(test_files_folder, folder_name)
+        if os.path.isdir(src_folder_path):
+            shutil.copytree(src_folder_path, dest_folder_path)
+            logging.info(f"Copied folder {src_folder_path} to {dest_folder_path}")
+
+def run_cleanup() -> None:
+    delete_random_test_case()
+    copy_default_folders()
+
+if __name__ == "__main__":
+    # Run this after running the script
+    delete_random_test_case()
+    copy_default_folders()

supporting_scripts/course-scripts/quick-course-setup/randomize_results_before.py

@@ -0,0 +1,29 @@
+import os
+import shutil
+from logging_config import logging
+
+test_files_folder: str = "../../../src/main/resources/templates/java/test/testFiles"
+random_test_case_file: str = "./testFiles-template/randomized/RandomizedTestCases.java"
+random_test_case_dest: str = os.path.join(test_files_folder, "RandomizedTestCases.java")
+
+def delete_existing_folders() -> None:
+    folders_to_delete: list[str] = ["behavior", "structural"]
+    for folder in folders_to_delete:
+        folder_path: str = os.path.join(test_files_folder, folder)
+        if os.path.exists(folder_path) and os.path.isdir(folder_path):
+            shutil.rmtree(folder_path)
+            logging.info(f"Deleted folder: {folder_path}")
+        else:
+            logging.info(f"Folder not found: {folder_path}")
+
+def copy_random_test_case() -> None:
+    if os.path.exists(random_test_case_file):
+        shutil.copy(random_test_case_file, random_test_case_dest)
+        logging.info(f"Copied {random_test_case_file} to {random_test_case_dest}")
+    else:
+        logging.info(f"RandomizedTestCases.java file not found at {random_test_case_file}")
+
+if __name__ == "__main__":
+    # Run this before running the script
+    delete_existing_folders()
+    copy_random_test_case()

supporting_scripts/course-scripts/quick-course-setup/testFiles-template/default/behavior/SortingExampleBehaviorTest.java

@@ -0,0 +1,98 @@
+package ${packageName};
+
+import org.junit.jupiter.api.*;
+import static org.junit.jupiter.api.Assertions.*;
+
+import java.lang.reflect.InvocationTargetException;
+import java.text.*;
+import java.util.*;
+
+import static de.tum.in.test.api.util.ReflectionTestUtils.*;
+
+import de.tum.in.test.api.BlacklistPath;
+import de.tum.in.test.api.PathType;
+import de.tum.in.test.api.StrictTimeout;
+import de.tum.in.test.api.WhitelistPath;
+import de.tum.in.test.api.jupiter.Public;
+
+/**
+ * @author Stephan Krusche (krusche@in.tum.de)
+ * @version 5.1 (11.06.2021)
+ */
+@Public
+@WhitelistPath("target") // mainly for Artemis
+@BlacklistPath("target/test-classes") // prevent access to test-related classes and resources
+class SortingExampleBehaviorTest {
+
+    private List<Date> dates;
+    private List<Date> datesWithCorrectOrder;
+
+    @BeforeEach
+    void setup() throws ParseException {
+        SimpleDateFormat dateFormat = new SimpleDateFormat("dd.MM.yyyy");
+        Date date1 = dateFormat.parse("08.11.2018");
+        Date date2 = dateFormat.parse("15.04.2017");
+        Date date3 = dateFormat.parse("15.02.2016");
+        Date date4 = dateFormat.parse("15.09.2017");
+
+        this.dates = Arrays.asList(date1, date2, date3, date4);
+        this.datesWithCorrectOrder = Arrays.asList(date3, date2, date4, date1);
+    }
+
+    @Test
+    @StrictTimeout(1)
+    void testBubbleSort() {
+        BubbleSort bubbleSort = new BubbleSort();
+        bubbleSort.performSort(dates);
+        if (!datesWithCorrectOrder.equals(dates)) {
+            fail("BubbleSort does not sort correctly");
+        }
+    }
+
+    @Test
+    @StrictTimeout(1)
+    void testMergeSort() {
+        MergeSort mergeSort = new MergeSort();
+        mergeSort.performSort(dates);
+        if (!datesWithCorrectOrder.equals(dates)) {
+            fail("MergeSort does not sort correctly");
+        }
+    }
+
+    @Test
+    @StrictTimeout(1)
+    void testUseMergeSortForBigList() throws ReflectiveOperationException {
+        List<Date> bigList = new ArrayList<Date>();
+        for (int i = 0; i < 11; i++) {
+            bigList.add(new Date());
+        }
+        Object chosenSortStrategy = configurePolicyAndContext(bigList);
+        if (!(chosenSortStrategy instanceof MergeSort)) {
+            fail("The sort algorithm of Context was not MergeSort for a list with more than 10 dates.");
+        }
+    }
+
+    @Test
+    @StrictTimeout(1)
+    void testUseBubbleSortForSmallList()  throws ReflectiveOperationException {
+        List<Date> smallList = new ArrayList<Date>();
+        for (int i = 0; i < 3; i++) {
+            smallList.add(new Date());
+        }
+        Object chosenSortStrategy = configurePolicyAndContext(smallList);
+        if (!(chosenSortStrategy instanceof BubbleSort)) {
+            fail("The sort algorithm of Context was not BubbleSort for a list with less or equal than 10 dates.");
+        }
+    }
+
+    private Object configurePolicyAndContext(List<Date> dates) throws ReflectiveOperationException {
+        Object context = newInstance("${packageName}.Context");
+        invokeMethod(context, getMethod(context, "setDates", List.class), dates);
+
+        Object policy = newInstance("${packageName}.Policy", context);
+        invokeMethod(policy, getMethod(policy, "configure"));
+
+        Object chosenSortStrategy = invokeMethod(context, getMethod(context, "getSortAlgorithm"));
+        return chosenSortStrategy;
+    }
+}

supporting_scripts/course-scripts/quick-course-setup/testFiles-template/default/structural/AttributeTest.java

@@ -0,0 +1,39 @@
+package ${packageName};
+
+import java.net.URISyntaxException;
+
+import org.junit.jupiter.api.DynamicContainer;
+import org.junit.jupiter.api.TestFactory;
+
+import de.tum.in.test.api.BlacklistPath;
+import de.tum.in.test.api.PathType;
+import de.tum.in.test.api.StrictTimeout;
+import de.tum.in.test.api.WhitelistPath;
+import de.tum.in.test.api.jupiter.Public;
+import de.tum.in.test.api.structural.AttributeTestProvider;
+
+/**
+ * @author Stephan Krusche (krusche@in.tum.de)
+ * @version 5.1 (11.06.2021)
+ * <br><br>
+ * This test evaluates if the specified attributes in the structure oracle are correctly implemented with the expected type, visibility modifiers and annotations,
+ * based on its definition in the structure oracle (test.json).
+ */
+@Public
+@WhitelistPath("target") // mainly for Artemis
+@BlacklistPath("target/test-classes") // prevent access to test-related classes and resources
+class AttributeTest extends AttributeTestProvider {
+
+    /**
+     * This method collects the classes in the structure oracle file for which attributes are specified.
+     * These classes are then transformed into JUnit 5 dynamic tests.
+     * @return A dynamic test container containing the test for each class which is then executed by JUnit.
+     */
+    @Override
+    @StrictTimeout(10)
+    @TestFactory
+    protected DynamicContainer generateTestsForAllClasses() throws URISyntaxException {
+        structureOracleJSON = retrieveStructureOracleJSON(this.getClass().getResource("test.json"));
+        return super.generateTestsForAllClasses();
+    }
+}

supporting_scripts/course-scripts/quick-course-setup/testFiles-template/default/structural/ClassTest.java

@@ -0,0 +1,39 @@
+package ${packageName};
+
+import java.net.URISyntaxException;
+
+import org.junit.jupiter.api.DynamicContainer;
+import org.junit.jupiter.api.TestFactory;
+
+import de.tum.in.test.api.BlacklistPath;
+import de.tum.in.test.api.PathType;
+import de.tum.in.test.api.StrictTimeout;
+import de.tum.in.test.api.WhitelistPath;
+import de.tum.in.test.api.jupiter.Public;
+import de.tum.in.test.api.structural.ClassTestProvider;
+
+/**
+ * @author Stephan Krusche (krusche@in.tum.de)
+ * @version 5.1 (11.06.2021)
+ * <br><br>
+ * This test evaluates the hierarchy of the class, i.e. if the class is abstract or an interface or an enum and also if the class extends another superclass and if
+ * it implements the interfaces and annotations, based on its definition in the structure oracle (test.json).
+ */
+@Public
+@WhitelistPath("target") // mainly for Artemis
+@BlacklistPath("target/test-classes") // prevent access to test-related classes and resources
+class ClassTest extends ClassTestProvider {
+
+    /**
+     * This method collects the classes in the structure oracle file for which at least one class property is specified.
+     * These classes are then transformed into JUnit 5 dynamic tests.
+     * @return A dynamic test container containing the test for each class which is then executed by JUnit.
+     */
+    @Override
+    @StrictTimeout(10)
+    @TestFactory
+    protected DynamicContainer generateTestsForAllClasses() throws URISyntaxException {
+        structureOracleJSON = retrieveStructureOracleJSON(this.getClass().getResource("test.json"));
+        return super.generateTestsForAllClasses();
+    }
+}

supporting_scripts/course-scripts/quick-course-setup/testFiles-template/default/structural/ConstructorTest.java

@@ -0,0 +1,39 @@
+package ${packageName};
+
+import java.net.URISyntaxException;
+
+import org.junit.jupiter.api.DynamicContainer;
+import org.junit.jupiter.api.TestFactory;
+
+import de.tum.in.test.api.BlacklistPath;
+import de.tum.in.test.api.PathType;
+import de.tum.in.test.api.StrictTimeout;
+import de.tum.in.test.api.WhitelistPath;
+import de.tum.in.test.api.jupiter.Public;
+import de.tum.in.test.api.structural.ConstructorTestProvider;
+
+/**
+ * @author Stephan Krusche (krusche@in.tum.de)
+ * @version 5.1 (11.06.2021)
+ * <br><br>
+ * This test evaluates if the specified constructors in the structure oracle are correctly implemented with the expected parameter types and annotations,
+ * based on its definition in the structure oracle (test.json).
+ */
+@Public
+@WhitelistPath("target") // mainly for Artemis
+@BlacklistPath("target/test-classes") // prevent access to test-related classes and resources
+class ConstructorTest extends ConstructorTestProvider {
+
+    /**
+     * This method collects the classes in the structure oracle file for which constructors are specified.
+     * These classes are then transformed into JUnit 5 dynamic tests.
+     * @return A dynamic test container containing the test for each class which is then executed by JUnit.
+     */
+    @Override
+    @StrictTimeout(10)
+    @TestFactory
+    protected DynamicContainer generateTestsForAllClasses() throws URISyntaxException {
+        structureOracleJSON = retrieveStructureOracleJSON(this.getClass().getResource("test.json"));
+        return super.generateTestsForAllClasses();
+    }
+}