Development: Fix GitLab CI Setup (#8023)

docker/gitlab-gitlabci.yml

@@ -4,34 +4,13 @@
 
 services:
     gitlab:
-        # TODO: check if more settings can also be integrated into gitlab.yml
         extends:
             file: ./gitlab.yml
             service: gitlab
-        volumes:
-            - artemis-gitlab-data:/var/opt/gitlab
-            - artemis-gitlab-logs:/var/log/gitlab
-            - artemis-gitlab-config:/etc/gitlab
         environment:
             GITLAB_OMNIBUS_CONFIG: |
-                external_url "${GIT_SERVER_NAME}"
-                prometheus_monitoring['enable'] = false
-                gitlab_rails['gitlab_shell_ssh_port'] = 2222
-                gitlab_rails['monitoring_whitelist'] = ['0.0.0.0/0']
-                letsencrypt['enable'] = ${SSL_ENABLED}
-                letsencrypt['auto_renew_hour'] = "12"
-                letsencrypt['auto_renew_minute'] = "30"
-                letsencrypt['auto_renew_day_of_month'] = "*/7"
-        ports:
-            - "2222:22"
-            - "80:80"
-            - "443:443"
-        # expose the ports to make them reachable docker internally even if the external port mapping changes
-        expose:
-            - "22"
-            - "80"
-            - "443"
-        shm_size: "256m"
+                external_url "http://host.docker.internal:8081"
+                nginx['listen_port'] = 80
     gitlab-runner:
         image: docker.io/gitlab/gitlab-runner:latest
         pull_policy: if_not_present

docs/dev/setup/gitlabci-gitlab.rst

@@ -28,62 +28,61 @@ For a production setup of GitLab, also see the documentation of the GitLab and J
 GitLab
 """"""
 
-1. Depending on your operating system, it is necessary to update the host file of your machine to include the following line:
+1. Depending on your operating system and your Docker installation, it is necessary to update the host file of your machine to include the following lines:
 
     .. code:: text
 
         127.0.0.1       host.docker.internal
         ::1             host.docker.internal
 
-2. Configure GitLab
+2. Start GitLab and the GitLab Runner
     .. code:: bash
 
-        cp docker/env.example.gitlab-gitlabci.txt docker/.env
+        docker-compose -f docker/gitlab-gitlabci.yml up -d
 
-3. Start GitLab and the GitLab Runner
+3. Get your GitLab root password
     .. code:: bash
 
-        docker-compose -f docker/gitlab-gitlabci.yml --env-file docker/.env up --build -d
+        docker exec artemis-gitlab grep 'Password:' /etc/gitlab/initial_root_password
 
-4. Get your GitLab root password
-    .. code:: bash
-
-        docker exec -it gitlab grep 'Password:' /etc/gitlab/initial_root_password
+4. Generate an access token
+    Go to ``http://localhost:8081/-/profile/personal_access_tokens`` and generate an access token with all scopes.
+    This token is used for generating a runner token and in the Artemis configuration as ``artemis.version-control.token``.
 
-5. Generate an access token
-    Go to ``http://host.docker.internal/-/profile/personal_access_tokens`` and generate an access token with all scopes.
-    This token is used in the Artemis configuration as ``artemis.version-control.token``.
-
-6. Allow outbound requests to local network
+5. Allow outbound requests to local network
     For setting up the webhook between Artemis and GitLab, it is necessary to allow requests to the local network.
-    Go to ``http://host.docker.internal/admin/application_settings/network`` and allow the outbound requests.
+    Go to ``http://localhost:8081/admin/application_settings/network`` and allow the outbound requests.
     More information about this aspect can be found in the `GitLab setup instructions <#gitlab-access-token>`__ (step 12).
 
 GitLab Runner
 """""""""""""
 
-1. Register a new runner
-    Login to your GitLab instance and open ``http://host.docker.internal/admin/runners``.
-    Click on ``Register an instance runner`` and copy the registration token.
+1. Create a runner token
+    In order to create a token which can be used to register a runner, go to ``http://localhost:8081/admin/runners/new``.
+    Alternatively you can execute the command below with the personal access token generated before.
 
+    .. code:: bash
+
+        curl --request POST --header "PRIVATE-TOKEN: <gitlab-personal-access-token>" \
+        --data "runner_type=instance_type" \
+        --data "run_untagged=true" \
+        --data "access_level=not_protected" \
+         "http://host.docker.internal:8081/api/v4/user/runners"
+
+1. Register a new runner
     Then execute this command with the registration token:
 
     .. code:: bash
 
-        docker exec -it gitlab-runner gitlab-runner register \
+        docker exec -it artemis-gitlab-runner gitlab-runner register \
         --non-interactive \
         --executor "docker" \
         --docker-image alpine:latest \
-        --url http://host.docker.internal:80 \
-        --registration-token "PROJECT_REGISTRATION_TOKEN" \
-        --description "docker-runner" \
-        --maintenance-note "Test Runner" \
-        --tag-list "docker,artemis" \
-        --run-untagged="true" \
-        --locked="false" \
-        --access-level="not_protected"
+        --url http://host.docker.internal:8081 \
+        --token "REGISTRATION_TOKEN" \
+        --description "docker-runner"
 
-    You should now find the runner in the list of runners (``http://host.docker.internal/admin/runners``)
+    You should now find the runner in the list of runners (``http://localhost:8081/admin/runners``)
 
 .. note::
     Adding a runner in a production setup works the same way.
@@ -116,7 +115,7 @@ Artemis
                 login:
                     account-name: TUM
             version-control:
-                url: http://host.docker.internal:80
+                url: http://localhost:8081
                 user: root
                 password: password # change this value
                 token: gitlab-personal-access-token # change this value

src/main/java/de/tum/in/www1/artemis/config/migration/entries/MigrationEntry20230808_203400.java

@@ -68,7 +68,7 @@ public class MigrationEntry20230808_203400 extends MigrationEntry {
 
     private final CopyOnWriteArrayList<ProgrammingExerciseParticipation> errorList = new CopyOnWriteArrayList<>();
 
-    private static final List<String> MIGRATABLE_PROFILES = List.of("bamboo", "gitlab", "jenkins");
+    private static final List<String> MIGRATABLE_PROFILES = List.of("bamboo", "jenkins");
 
     public MigrationEntry20230808_203400(ProgrammingExerciseRepository programmingExerciseRepository,
             SolutionProgrammingExerciseParticipationRepository solutionProgrammingExerciseParticipationRepository,

src/main/java/de/tum/in/www1/artemis/service/connectors/gitlabci/GitLabCIProgrammingLanguageFeatureService.java

@@ -16,7 +16,7 @@
 public class GitLabCIProgrammingLanguageFeatureService extends ProgrammingLanguageFeatureService {
 
     public GitLabCIProgrammingLanguageFeatureService() {
-        programmingLanguageFeatures.put(EMPTY, new ProgrammingLanguageFeature(EMPTY, false, false, false, false, false, List.of(), false, true, false));
-        programmingLanguageFeatures.put(JAVA, new ProgrammingLanguageFeature(JAVA, false, false, false, true, false, List.of(PLAIN_MAVEN, MAVEN_MAVEN), true, true, false));
+        programmingLanguageFeatures.put(EMPTY, new ProgrammingLanguageFeature(EMPTY, false, false, false, false, false, List.of(), false, false, false));
+        programmingLanguageFeatures.put(JAVA, new ProgrammingLanguageFeature(JAVA, false, false, false, true, false, List.of(PLAIN_MAVEN, MAVEN_MAVEN), false, false, false));
     }
 }

src/main/java/de/tum/in/www1/artemis/service/connectors/gitlabci/GitLabCIService.java

@@ -40,6 +40,8 @@
 @Service
 public class GitLabCIService extends AbstractContinuousIntegrationService {
 
+    private static final String GITLAB_CI_FILE_EXTENSION = ".yml";
+
     private static final Logger log = LoggerFactory.getLogger(GitLabCIService.class);
 
     private static final String VARIABLE_BUILD_DOCKER_IMAGE_NAME = "ARTEMIS_BUILD_DOCKER_IMAGE";
@@ -112,7 +114,7 @@ public GitLabCIService(ProgrammingSubmissionRepository programmingSubmissionRepo
     public void createBuildPlanForExercise(ProgrammingExercise exercise, String planKey, VcsRepositoryUri repositoryUri, VcsRepositoryUri testRepositoryUri,
             VcsRepositoryUri solutionRepositoryUri) {
         addBuildPlanToProgrammingExerciseIfUnset(exercise);
-        setupGitLabCIConfiguration(repositoryUri, exercise, planKey);
+        setupGitLabCIConfiguration(repositoryUri, exercise, generateBuildPlanId(exercise.getProjectKey(), planKey));
         // TODO: triggerBuild(repositoryUri, exercise.getBranch());
     }
 
@@ -126,7 +128,7 @@ private void setupGitLabCIConfiguration(VcsRepositoryUri repositoryUri, Programm
             project.setSharedRunnersEnabled(true);
             project.setAutoDevopsEnabled(false);
 
-            final String buildPlanUrl = buildPlanService.generateBuildPlanURL(exercise);
+            final String buildPlanUrl = buildPlanService.generateBuildPlanURL(exercise) + "&file-extension=" + GITLAB_CI_FILE_EXTENSION;
             project.setCiConfigPath(buildPlanUrl);
 
             projectApi.updateProject(project);
@@ -200,7 +202,11 @@ public String copyBuildPlan(ProgrammingExercise sourceExercise, String sourcePla
 
         // When sending the build results back, the build plan key is used to identify the participation.
         // Therefore, we return the key here even though GitLab CI does not need it.
-        return targetExercise.getProjectKey() + "-" + targetPlanName.toUpperCase().replaceAll("[^A-Z0-9]", "");
+        return generateBuildPlanId(targetExercise.getProjectKey(), targetPlanName);
+    }
+
+    private String generateBuildPlanId(String projectKey, String planKey) {
+        return projectKey + "-" + planKey.toUpperCase().replaceAll("[^A-Z0-9]", "");
     }
 
     @Override