Integrated code lifecycle: Provide Instructors more options to control container configuration

docs/user/exercises/programming-exercise-setup.inc

@@ -404,14 +404,39 @@ Edit Maximum Build Duration
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 **This option is only available when using** :ref:`integrated code lifecycle<integrated code lifecycle>`
-This section is optional. In most cases, the preconfigured build script does not need to be changed.
+
+This section is optional. In most cases, the default maximum build duration does not need to be changed.
 
 The maximum build duration is the time limit for the build plan to execute. If the build plan exceeds this time limit, it will be terminated. The default value is 120 seconds.
 You can change the maximum build duration by using the slider.
 
 .. figure:: programming/timeout-slider.png
           :align: center
 
+Edit Container Configuration
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+**This option is only available when using** :ref:`integrated code lifecycle<integrated code lifecycle>`
+
+This section is optional. In most cases, the default container configuration does not need to be changed.
+
+Currently, instructors can only change whether the container has internet access and add additional environment variables.
+Disabling internet access can be useful if instructors want to prevent students from downloading additional dependencies during the build process.
+If internet access is disabled, the container cannot access the internet during the build process. Thus, it will not be able to download additional dependencies.
+The dependencies must then be included/cached in the docker image.
+
+Additional environment variables can be added to the container configuration. This can be useful if the build process requires additional environment variables to be set.
+Keys and values can be wrapped in single or double quotes. The keys and values must be separated by an equal sign. Multiple key-value pairs can be added by separating them with a comma with no space in between.
+
+.. figure:: programming/docker-flags-edit.png
+          :align: center
+
+We plan to add more options to the container configuration in the future.
+
+.. warning::
+  - Disabling internet access is not currently supported for Swift and Haskell exercises.
+
+
 .. _configure_static_code_analysis_tools:
 
 Configure static code analysis

src/main/java/de/tum/cit/aet/artemis/buildagent/dto/BuildConfig.java

@@ -15,7 +15,8 @@
 @JsonInclude(JsonInclude.Include.NON_EMPTY)
 public record BuildConfig(String buildScript, String dockerImage, String commitHashToBuild, String assignmentCommitHash, String testCommitHash, String branch,
         ProgrammingLanguage programmingLanguage, ProjectType projectType, boolean scaEnabled, boolean sequentialTestRunsEnabled, boolean testwiseCoverageEnabled,
-        List<String> resultPaths, int timeoutSeconds, String assignmentCheckoutPath, String testCheckoutPath, String solutionCheckoutPath) implements Serializable {
+        List<String> resultPaths, int timeoutSeconds, String assignmentCheckoutPath, String testCheckoutPath, String solutionCheckoutPath, DockerRunConfig dockerRunConfig)
+        implements Serializable {
 
     @Override
     public String dockerImage() {

src/main/java/de/tum/cit/aet/artemis/buildagent/dto/DockerRunConfig.java

@@ -0,0 +1,36 @@
+package de.tum.cit.aet.artemis.buildagent.dto;
+
+import java.io.Serializable;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+public record DockerRunConfig(boolean isNetworkDisabled, List<String> env) implements Serializable {
+
+    public enum AllowedDockerFlags {
+
+        NETWORK("network"), ENV("env");
+
+        private final String flag;
+
+        AllowedDockerFlags(String flag) {
+            this.flag = flag;
+        }
+
+        public String flag() {
+            return flag;
+        }
+
+        private static final Set<String> ALLOWED_FLAGS = new HashSet<>();
+
+        static {
+            for (AllowedDockerFlags value : values()) {
+                ALLOWED_FLAGS.add(value.flag());
+            }
+        }
+
+        public static boolean isAllowed(String flag) {
+            return ALLOWED_FLAGS.contains(flag);
+        }
+    }
+}

src/main/java/de/tum/cit/aet/artemis/buildagent/service/BuildJobContainerService.java

@@ -85,19 +85,23 @@ public BuildJobContainerService(DockerClient dockerClient, HostConfig hostConfig
     /**
      * Configure a container with the Docker image, the container name, optional proxy config variables, and set the command that runs when the container starts.
      *
-     * @param containerName the name of the container to be created
-     * @param image         the Docker image to use for the container
-     * @param buildScript   the build script to be executed in the container
+     * @param containerName   the name of the container to be created
+     * @param image           the Docker image to use for the container
+     * @param buildScript     the build script to be executed in the container
+     * @param exerciseEnvVars the environment variables provided by the instructor
      * @return {@link CreateContainerResponse} that can be used to start the container
      */
-    public CreateContainerResponse configureContainer(String containerName, String image, String buildScript) {
+    public CreateContainerResponse configureContainer(String containerName, String image, String buildScript, List<String> exerciseEnvVars) {
         List<String> envVars = new ArrayList<>();
         if (useSystemProxy) {
             envVars.add("HTTP_PROXY=" + httpProxy);
             envVars.add("HTTPS_PROXY=" + httpsProxy);
             envVars.add("NO_PROXY=" + noProxy);
         }
         envVars.add("SCRIPT=" + buildScript);
+        if (exerciseEnvVars != null && !exerciseEnvVars.isEmpty()) {
+            envVars.addAll(exerciseEnvVars);
+        }
         return dockerClient.createContainerCmd(image).withName(containerName).withHostConfig(hostConfig).withEnv(envVars)
                 // Command to run when the container starts. This is the command that will be executed in the container's main process, which runs in the foreground and blocks the
                 // container from exiting until it finishes.
@@ -121,11 +125,21 @@ public void startContainer(String containerId) {
     /**
      * Run the script in the container and wait for it to finish before returning.
      *
-     * @param containerId the id of the container in which the script should be run
-     * @param buildJobId  the id of the build job that is currently being executed
+     * @param containerId       the id of the container in which the script should be run
+     * @param buildJobId        the id of the build job that is currently being executed
+     * @param isNetworkDisabled whether the network should be disabled for the container
      */
+    public void runScriptInContainer(String containerId, String buildJobId, boolean isNetworkDisabled) {
+        if (isNetworkDisabled) {
+            log.info("disconnecting container with id {} from network", containerId);
+            try {
+                dockerClient.disconnectFromNetworkCmd().withContainerId(containerId).withNetworkId("bridge").exec();
+            }
+            catch (Exception e) {
+                log.error("Failed to disconnect container with id {} from network: {}", containerId, e.getMessage());
+            }
+        }
 
-    public void runScriptInContainer(String containerId, String buildJobId) {
         log.info("Started running the build script for build job in container with id {}", containerId);
         // The "sh script.sh" execution command specified here is run inside the container as an additional process. This command runs in the background, independent of the
         // container's
@@ -448,9 +462,4 @@ private Container getContainerForName(String containerName) {
         List<Container> containers = dockerClient.listContainersCmd().withShowAll(true).exec();
         return containers.stream().filter(container -> container.getNames()[0].equals("/" + containerName)).findFirst().orElse(null);
     }
-
-    private String getParentFolderPath(String path) {
-        Path parentPath = Paths.get(path).normalize().getParent();
-        return parentPath != null ? parentPath.toString() : "";
-    }
 }

src/main/java/de/tum/cit/aet/artemis/buildagent/service/BuildJobExecutionService.java

@@ -232,10 +232,18 @@ public BuildResult runBuildJob(BuildJobQueueItem buildJob, String containerName)
             index++;
         }
 
-        CreateContainerResponse container = buildJobContainerService.configureContainer(containerName, buildJob.buildConfig().dockerImage(), buildJob.buildConfig().buildScript());
+        List<String> envVars = null;
+        boolean isNetworkDisabled = false;
+        if (buildJob.buildConfig().dockerRunConfig() != null) {
+            envVars = buildJob.buildConfig().dockerRunConfig().env();
+            isNetworkDisabled = buildJob.buildConfig().dockerRunConfig().isNetworkDisabled();
+        }
+
+        CreateContainerResponse container = buildJobContainerService.configureContainer(containerName, buildJob.buildConfig().dockerImage(), buildJob.buildConfig().buildScript(),
+                envVars);
 
         return runScriptAndParseResults(buildJob, containerName, container.getId(), assignmentRepoUri, testsRepoUri, solutionRepoUri, auxiliaryRepositoriesUris,
-                assignmentRepositoryPath, testsRepositoryPath, solutionRepositoryPath, auxiliaryRepositoriesPaths, assignmentCommitHash, testCommitHash);
+                assignmentRepositoryPath, testsRepositoryPath, solutionRepositoryPath, auxiliaryRepositoriesPaths, assignmentCommitHash, testCommitHash, isNetworkDisabled);
     }
 
     /**
@@ -270,7 +278,7 @@ public BuildResult runBuildJob(BuildJobQueueItem buildJob, String containerName)
     private BuildResult runScriptAndParseResults(BuildJobQueueItem buildJob, String containerName, String containerId, VcsRepositoryUri assignmentRepositoryUri,
             VcsRepositoryUri testRepositoryUri, VcsRepositoryUri solutionRepositoryUri, VcsRepositoryUri[] auxiliaryRepositoriesUris, Path assignmentRepositoryPath,
             Path testsRepositoryPath, Path solutionRepositoryPath, Path[] auxiliaryRepositoriesPaths, @Nullable String assignmentRepoCommitHash,
-            @Nullable String testRepoCommitHash) {
+            @Nullable String testRepoCommitHash, boolean isNetworkDisabled) {
 
         long timeNanoStart = System.nanoTime();
 
@@ -292,7 +300,7 @@ private BuildResult runScriptAndParseResults(BuildJobQueueItem buildJob, String
         buildLogsMap.appendBuildLogEntry(buildJob.id(), msg);
         log.debug(msg);
 
-        buildJobContainerService.runScriptInContainer(containerId, buildJob.id());
+        buildJobContainerService.runScriptInContainer(containerId, buildJob.id(), isNetworkDisabled);
 
         msg = "~~~~~~~~~~~~~~~~~~~~ Finished Executing Build Script for Build job " + buildJob.id() + " ~~~~~~~~~~~~~~~~~~~~";
         buildLogsMap.appendBuildLogEntry(buildJob.id(), msg);

src/main/java/de/tum/cit/aet/artemis/core/config/Constants.java

@@ -39,6 +39,8 @@ public final class Constants {
 
     public static final int QUIZ_GRACE_PERIOD_IN_SECONDS = 5;
 
+    public static final int MAX_ENVIRONMENT_VARIABLES_DOCKER_FLAG_LENGTH = 1000;
+
     /**
      * This constant determines how many seconds after the exercise due dates submissions will still be considered rated.
      * Submissions after the grace period exceeded will be flagged as illegal.

src/main/java/de/tum/cit/aet/artemis/programming/domain/ProgrammingExerciseBuildConfig.java

@@ -60,7 +60,7 @@ public class ProgrammingExerciseBuildConfig extends DomainObject {
     @Column(name = "timeout_seconds")
     private int timeoutSeconds;
 
-    @Column(name = "docker_flags")
+    @Column(name = "docker_flags", columnDefinition = "longtext")
     private String dockerFlags;
 
     @OneToOne(mappedBy = "buildConfig")

src/main/java/de/tum/cit/aet/artemis/programming/service/ProgrammingExerciseBuildConfigService.java

@@ -0,0 +1,161 @@
+package de.tum.cit.aet.artemis.programming.service;
+
+import static de.tum.cit.aet.artemis.core.config.Constants.MAX_ENVIRONMENT_VARIABLES_DOCKER_FLAG_LENGTH;
+import static de.tum.cit.aet.artemis.core.config.Constants.PROFILE_CORE;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import jakarta.annotation.Nullable;
+
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.springframework.context.annotation.Profile;
+import org.springframework.stereotype.Service;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import de.tum.cit.aet.artemis.buildagent.dto.DockerRunConfig;
+import de.tum.cit.aet.artemis.programming.domain.ProgrammingExerciseBuildConfig;
+
+@Profile(PROFILE_CORE)
+@Service
+public class ProgrammingExerciseBuildConfigService {
+
+    private static final Logger log = org.slf4j.LoggerFactory.getLogger(ProgrammingExerciseBuildConfigService.class);
+
+    /**
+     * Converts a JSON string representing Docker flags (in the form of a list of key-value pairs)
+     * into a {@link DockerRunConfig} instance.
+     *
+     * <p>
+     * The JSON string is expected to represent a list of key-value pairs where each
+     * entry is a list containing two strings: the first being the key and the second being the value.
+     * Example JSON input:
+     *
+     * <pre>
+     * [["network", "none"], ["env", "TEST"]]
+     * </pre>
+     *
+     * @param buildConfig the build config containing the Docker flags
+     * @return a {@link DockerRunConfig} object initialized with the parsed flags, or {@code null} if an error occurs
+     */
+    @Nullable
+    public DockerRunConfig getDockerRunConfig(ProgrammingExerciseBuildConfig buildConfig) {
+        List<List<String>> parsedList = parseDockerFlags(buildConfig);
+        if (parsedList == null) {
+            return null;
+        }
+        return getDockerRunConfigFromParsedList(parsedList);
+    }
+
+    /**
+     * Converts a list of key-value pairs representing Docker flags into a {@link DockerRunConfig} instance. @see {@link #getDockerRunConfig(ProgrammingExerciseBuildConfig)}
+     *
+     * @param list the list of key-value pairs
+     * @return a {@link DockerRunConfig} object initialized with the parsed flags, or {@code null} if an error occurs
+     */
+    @Nullable
+    DockerRunConfig getDockerRunConfigFromParsedList(List<List<String>> list) {
+        final int keyIndex = 0;
+        final int valueIndex = 1;
+        try {
+            boolean networkDisabled = false;
+            List<String> env = null;
+            for (List<String> entry : list) {
+                if (entry.size() != 2 || StringUtils.isBlank(entry.get(valueIndex)) || StringUtils.isBlank(entry.get(keyIndex))
+                        || !DockerRunConfig.AllowedDockerFlags.isAllowed(entry.get(keyIndex))) {
+                    log.warn("Invalid Docker flag entry: {}. Skipping.", entry);
+                    continue;
+                }
+                switch (entry.get(keyIndex)) {
+                    case "network":
+                        networkDisabled = entry.get(valueIndex).equalsIgnoreCase("none");
+                        break;
+                    case "env":
+                        env = parseEnvVariableString(entry.get(valueIndex));
+                        break;
+                    default:
+                        log.error("Invalid Docker flag entry: {}. Skipping.", entry);
+                        break;
+                }
+
+            }
+            return new DockerRunConfig(networkDisabled, env);
+        }
+        catch (Exception e) {
+            log.error("Failed to parse DockerRunConfig from JSON string: {}. Using default settings.", list, e);
+        }
+
+        return null;
+    }
+
+    /**
+     * Parses the JSON string representing Docker flags into a list of key-value pairs.
+     *
+     * @return a list of key-value pairs, or {@code null} if an error occurs
+     */
+    @Nullable
+    List<List<String>> parseDockerFlags(ProgrammingExerciseBuildConfig buildConfig) {
+        if (StringUtils.isBlank(buildConfig.getDockerFlags())) {
+            return null;
+        }
+
+        try {
+            ObjectMapper objectMapper = new ObjectMapper();
+
+            return objectMapper.readValue(buildConfig.getDockerFlags(), new TypeReference<>() {
+            });
+        }
+        catch (Exception e) {
+            log.error("Failed to parse DockerRunConfig from JSON string: {}. Using default settings.", buildConfig.getDockerFlags(), e);
+        }
+
+        return null;
+    }
+
+    private List<String> parseEnvVariableString(String envVariableString) {
+        Pattern pattern = Pattern.compile(
+                // match key-value pairs, where the key can be a single word or a string in single or double quotes
+                // key-value pairs are separated by commas
+                "(?:'([^']+)'|\"([^\"]+)\"|(\\w+))=(?:'([^']*)'|\"([^\"]*)\"|([^,]+))");
+
+        if (envVariableString.length() > MAX_ENVIRONMENT_VARIABLES_DOCKER_FLAG_LENGTH) {
+            log.warn("The environment variables string is too long. It will be truncated to {} characters.", MAX_ENVIRONMENT_VARIABLES_DOCKER_FLAG_LENGTH);
+            envVariableString = envVariableString.substring(0, MAX_ENVIRONMENT_VARIABLES_DOCKER_FLAG_LENGTH);
+        }
+
+        Matcher matcher = pattern.matcher(envVariableString);
+
+        return extractEnvVariablesKeyValues(matcher);
+    }
+
+    /**
+     * Extracts the key-value pairs from the matcher and returns them as a list of strings
+     * The key/value can be a single word, a string in single quotes, or a string in double quotes
+     *
+     * @param matcher the matcher that contains the key-value pairs
+     * @return a list of strings containing the key-value pairs
+     */
+    private List<String> extractEnvVariablesKeyValues(Matcher matcher) {
+        List<String> envVars = new ArrayList<>();
+        while (matcher.find()) {
+            // The key can be a single word, a string in single quotes, or a string in double quotes, if matched to group1, the key is in single quotes, if matched to group2, the
+            // key is in double quotes, otherwise it is a single word
+            // if all groups are null, the key is a single word
+            String key = matcher.group(1) != null ? matcher.group(1) : matcher.group(2) != null ? matcher.group(2) : matcher.group(3);
+
+            // The value can be a single word, a string in single quotes, or a string in double quotes, if matched to group4, the value is in single quotes, if matched to group5,
+            // the value is in double quotes, otherwise it is a single word
+            // if all groups are null, the value is a single word
+            String value = matcher.group(4) != null ? matcher.group(4) : matcher.group(5) != null ? matcher.group(5) : matcher.group(6);
+
+            // Add the key-value pair to the list
+            envVars.add(key + "=" + value);
+        }
+        return envVars;
+    }
+}