Add support for custom reload cmd & option to update entrypoints

.gitlab-ci.yml

@@ -16,45 +16,48 @@ include:
 # Defines .base job template.
 - project: frontier/cloud-hosted-beta/eoaas-deployment-pipeline
   file: gitlab-ci-templates/base.gitlab-ci.yml
-  ref: usr/dr/gmtxm
+  ref: main
 # Defines .crane and .crane-copy-multi-image templates.
 - project: frontier/cloud-hosted-beta/eoaas-deployment-pipeline
   file: gitlab-ci-templates/crane.gitlab-ci.yml
-  ref: usr/dr/gmtxm
+  ref: main
 
 variables:
   # Variables with descriptions so that a nice form is generated in the Gitlab UI
   # when a pipeline gets triggered.
   DEVENV_NAME:
     description: Name of the DevEnv that will be created.
     value: ""
-  EO_ENV:
-    description: Name of the EO deployment that will be targeted
+  APP_ENV:
+    description: Name of the app deployment that will be targeted.
     value: ""
   NAMESPACE:
-    description: Namespace where to deploy the DevEnv and look for the target services
-    value: hzp
+    description: Namespace where to deploy the DevEnv and look for the target services.
+    value: default
   SELECTORS:
-    description: Labels of target EO services to develop for
+    description: Labels of target EO services to develop for.
     value: app=api-v2
   MOUNT_PATH:
-    description: The path in the target pod were the code will be mounted at
+    description: The path in the target pod were the code will be mounted at.
     value: "/mist"
   MODE:
-    description: Choose if target pod should be modified or cloned
+    description: Choose if target pod should be modified or cloned.
     value: "modify"
     options: ["modify", "clone"]
   PORT:
-    description: The port of the target pod that should be exposed when using the clone mode
+    description: The port of the target pod that should be exposed when using the clone mode.
     value: "8080"
   RELOAD_SIGNAL:
-    description: The Unix signal to send to the target pod to reload the code
+    description: The Unix signal to send to the target pod to reload the code.
     value: "HUP"
     options: ["TERM", "HUP"]
+  RELOAD_CMD:
+    description: The command to run to target pods when reloading the code.
+    value: ""
   POST_MOUNT_POD_CMD:
-    description: The command to run after the target pod mounts the code
+    description: The command to run after the target pod mounts the code.
   EXCLUDED:
-    description: Excluded paths
+    description: Excluded paths.
     value: '[".git", ".vscode", "__pycache__", ".pyc", "/landing", "/portal", "/jenkins", "/settings", "/ui", "/tests", "/docker/nginx/static"]'
 
   # Remaining variables that will not be rendered in the Gitlab manual pipeline form.
@@ -111,16 +114,16 @@ deploy operator:
     GIT_STRATEGY: fetch
   stage: deploy operator
   rules:
-    - if: $EO_ENV == ""
+    - if: $APP_ENV == ""
       when: manual
     - when: always
   script:
   - |-
-    if [ -z "$EO_ENV" ]; then
+    if [ -z "$APP_ENV" ]; then
       echo "Deploying operator on host cluster"
     else
-      echo "Deploying operator inside $EO_ENV vcluster"
-      vcluster connect $EO_ENV
+      echo "Deploying operator inside $APP_ENV vcluster"
+      vcluster connect $APP_ENV
       sleep 5
     fi
     kubectl cluster-info
@@ -144,23 +147,28 @@ deploy devenv:
   needs:
   - deploy operator
   rules:
-    - if: $EO_ENV == ""
+    - if: $APP_ENV == ""
       when: manual
     - if: $DEVENV_NAME == ""
       when: manual
     - when: always
   script:
   - |-
-    echo "Deploying DevEnv"
-    vcluster connect $EO_ENV
-    echo "Connected to vcluster"
-    kubectl cluster-info
+    if [ -z "$APP_ENV" ]; then
+      echo "Deploying DevEnv on host cluster"
+    else
+      echo "Deploying DevEnv"
+      vcluster connect $APP_ENV
+      echo "Connected to vcluster"
+      kubectl cluster-info
+    fi
     echo "Excluded:" $EXCLUDED
     echo "Mount path:" $MOUNT_PATH
     echo "Selectors:" $SELECTORS
     echo "Mode:" $MODE
     echo "Port:" $PORT
     echo "Reload signal:" $RELOAD_SIGNAL
+    echo "Reload command:" $RELOAD_CMD
     echo "Post mount command:" $POST_MOUNT_POD_CMD
     curl https://gitlab.dell.com/$GITLAB_USER_LOGIN.keys > user.keys
     export GITHUB_USER_LOGIN=$(echo $GITLAB_USER_LOGIN|tr '_' '-')
@@ -170,6 +178,7 @@ deploy devenv:
     export MOUNTS=""
     export KIND="deployment"
     export GROUP_NAME="eoaas-development"
+    export ENTRYPOINTS="{}"
     for SELECTOR in $SELECTORS
         do
           export LABELS=$(echo $SELECTOR |sed -r 's/=/:\ /g'|sed -r 's/,/\n/g'|sed -r 's/^/      /g'|sed '1i\\')

.pre-commit-config.yaml

@@ -13,7 +13,7 @@ repos:
     - --branch=main
     - --pattern=^rel/
   - id: check-added-large-files
-    args: ["--maxkb=50"]
+    args: ["--maxkb=360"]
   - id: check-merge-conflict
   - id: check-case-conflict
   - id: fix-byte-order-marker

README.md

@@ -1,6 +1,8 @@
 # Remote Development Operator
 
-The Remote Development Operator, is a Kubernetes operator and CRD for deploying remote development environments. It has been developed within Dell ISG Edge, to facilitate the development efforts of __[Dell NativeEdge](https://www.dell.com/en-us/dt/solutions/edge-computing/edge-platform.htm)__, the edge operations platform. The Remote Development Operator can be leveraged to facilitate the development of any software project that can be deployed on Kubernetes.
+The Remote Development Operator, is a Kubernetes operator and Custom Resource Definition (CRD) for deploying remote development environments. It has been developed within Dell ISG Edge, to facilitate the development efforts of __[Dell NativeEdge](https://www.dell.com/en-us/dt/solutions/edge-computing/edge-platform.htm)__, the edge operations platform. The Remote Development Operator can be leveraged to facilitate the development of any software project that can be deployed on Kubernetes.
+
+<img src="Operator.png" />
 
 ## Table of Contents
 
@@ -12,11 +14,20 @@ The Remote Development Operator, is a Kubernetes operator and CRD for deploying
 - [License](#license)
 
 # Features
-The CRD, defines a new Kubernetes resource named __DevEnv__. Each __DevEnv__ can target one or more k8s deployments or statefulsets using label selectors. For each __DevEnv__, the operator creates a DNS record, a PVC for storing the code or binaries, and starts an SSH server, configured to accept the specified SSH keypairs. Each __DevEnv__, provides a command that can be used to configure the end user's IDE (e.g. VSCode). This configuration will synchronize over SSH the local code that's stored on the developer's device to the PVC that's attached to the __DevEnv__ pod. After the first sync is complete, the __DevEnv__ gets enabled and mounts the code PVC to the target pods.
+We define a new Kubernetes custom resource named __DevEnv__. Each __DevEnv__ can target one or more k8s deployments or statefulsets using label selectors. For each __DevEnv__, the operator creates a DNS record, a PVC for storing the code or binaries, and starts an SSH server, configured to accept the specified SSH keypairs. Each __DevEnv__, provides a command that can be used to configure the end user's IDE (e.g. VSCode). This configuration will synchronize over SSH the local code that's stored on the developer's device to the PVC that's attached to the __DevEnv__ pod. After the first sync is complete, the __DevEnv__ gets enabled and mounts the code PVC to the target pods.
 
 There are two supported modes of operation.
-- In `modify` mode the operator will edit the definition of the target deployments or statefulsets, mounting the code PVC on the target path, which will override the original code that's burned into the image.
-- In `clone` mode the operator will launch a new deployment or statefulset, with the PVC mounted on the target path, while leaving the original deployment or statefulset untouched. A new service and ingress is also generated, to expose a port of the cloned deployment over HTTPS.
+### Modify mode
+
+<img src="modify-mode.svg" width="600" />
+
+In `modify` mode the operator will edit the definition of the target deployments or statefulsets, mounting the code PVC on the target path, which will override the original code that's burned into the image.
+
+## Clone mode
+
+<img src="clone-mode.svg" width="600" />
+
+In `clone` mode the operator will launch a new deployment or statefulset, with the PVC mounted on the target path, while leaving the original deployment or statefulset untouched. A new service and ingress is also generated, to expose a port of the cloned deployment over HTTPS.
 
 When working on a large application that consists of numerous microservices, the `clone` mode allows multiple developers to work in parallel, leveraging the same deployed application. There may still be cases were the cloned devenv may affect the original application, e.g. when working on DB schema changes. Developers that leverage the `clone` mode should coordinate with the peers to ensure they're not breaking their development environments. When in doubt, `modify` is the safest option, however it mostly requires a dedicated application deployment for each developer.
 
@@ -50,6 +61,7 @@ Once the operator is installed, you can start creating DevEnvs. Copy examples/de
 | mounts | A list of mounts |
 | excludedPaths | The repository paths to be excluded from syncing. |
 | reloadSignal | The UNIX signal that will force the target resource to reload its code. Can be `TERM` or `HUP`. |
+| reloadCmd | A command to run on the pod of the target resource when reloading the code. |
 | postMountPodCmd | A command to run on the pod of the target resource after mounting the code PVC. |
 ||
 

devenv/.bashrc

@@ -81,13 +81,5 @@ if [ -f ~/.bash_aliases ]; then
     . ~/.bash_aliases
 fi
 
-# enable programmable completion features (you don't need to enable
-# this, if it's already enabled in /etc/bash.bashrc and /etc/profile
-# sources /etc/bash.bashrc).
-if ! shopt -oq posix; then
-  if [ -f /usr/share/bash-completion/bash_completion ]; then
-    . /usr/share/bash-completion/bash_completion
-  elif [ -f /etc/bash_completion ]; then
-    . /etc/bash_completion
-  fi
-fi
+# enable programmable completion features
+source /etc/bash_completion

devenv/Dockerfile

@@ -4,7 +4,7 @@ RUN useradd -ms /bin/bash $USER
 ENV HOME /home/$USER
 USER root
 # RUN adduser $USER
-RUN apt-get update && apt-get install jq curl fuse sudo sed apt-utils vim openssh-server gzip git rsync -y
+RUN apt-get update && apt-get install jq curl fuse sudo sed apt-utils vim openssh-server gzip git rsync bash-completion -y
 RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/`curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt`/bin/linux/amd64/kubectl && \
       chmod +x ./kubectl && \
       mv ./kubectl /usr/local/bin/kubectl && \

devenv/scripts/reload.sh

@@ -1,9 +1,35 @@
 #!/bin/bash
 
 # Send reload signal to target pods.
-for pod in `kubectl get po -lapp=api-v2 -o name`
+export pods=$(python -d <<EOF
+import os
+import yaml
+import base64
+import subprocess
+
+mounts = base64.b64decode(os.getenv('mounts'))
+mounts_yaml = f"""mounts:
+{mounts.decode()}"""
+mounts_obj = yaml.safe_load(mounts)
+
+for mount in mounts_obj:
+    labels = mount['labels']
+    labels_str = ",".join(f"{key}={val}" for key, val in labels.items())
+    print(subprocess.check_output(['kubectl','get','po',f"-l{labels_str}",'-o','name']).decode('utf-8'))
+EOF
+)
+
+for pod in $pods
 do
+    echo $pod
     kubectl exec -it $pod -- kill -$reload_signal 1
+    if [ -z "$reload_cmd" ]; then
+        # No reload command to be executed inside the pod.
+        echo
+    else
+        echo "Executing reload command inside $pod: $reload_cmd"
+        kubectl exec -it $pod -- $reload_cmd
+    fi
     echo "Reloading $pod"
 done
 
@@ -17,7 +43,7 @@ else
         echo "No post mount command to be executed inside the pods"
     else
         echo "Executing post mount command inside each pod"
-        for pod in `kubectl get po -lapp=api-v2 -o name`
+        for pod in $pods
         do
             echo kubectl exec -it $pod -- $post_mount_pod_cmd
             kubectl exec -it $pod -- $post_mount_pod_cmd

examples/devenv-template.yaml

@@ -13,4 +13,5 @@ spec:
   mounts: ${MOUNTS}
   excludedPaths: ${EXCLUDED}
   reloadSignal: ${RELOAD_SIGNAL}
+  reloadCmd: "${RELOAD_CMD}"
   postMountPodCmd: "${POST_MOUNT_POD_CMD}"

examples/mounts-template.yaml

@@ -1,3 +1,6 @@
   - kind: ${KIND}
     mountPath: ${MOUNT_PATH}
+    subPath: ${SUB_PATH}
     labels: ${LABELS}
+    entrypoints:
+      ${ENTRYPOINTS}

operator/crd.yaml

@@ -107,6 +107,11 @@ spec:
                       description: The labels that have been assigned to target Deployment or Statefulset.
                       additionalProperties:
                         type: string
+                    entrypoints:
+                      type: object
+                      description: Override the entrypoint of specific containers in the target Deployment or Statefulset when mounting the code volume.
+                      additionalProperties:
+                        type: string
                     mounted:
                       type: boolean
                       description: Whether the volume should actually be mounted.
@@ -118,6 +123,10 @@ spec:
                 enum:
                 - TERM
                 - HUP
+              reloadCmd:
+                type: string
+                description: Command to be executed to target pods when reloading the code.
+                default: ''
               postMountPodCmd:
                 type: string
                 description: Command to be executed to target pods after mounting the code volume.

operator/op.py

@@ -3,6 +3,7 @@
 import base64
 import functools
 import os
+import shlex
 import subprocess
 from copy import deepcopy
 
@@ -33,6 +34,7 @@ def create_update_dev_env(name, spec, namespace, logger, **kwargs):
     excluded_paths = spec["excludedPaths"]
     mounts = spec["mounts"]
     reload_signal = spec["reloadSignal"]
+    reload_cmd = spec["reloadCmd"]
     post_mount_pod_cmd = spec["postMountPodCmd"]
 
     # Interpolate all templates and apply idempotently using kubectl apply -f -
@@ -53,6 +55,7 @@ def create_update_dev_env(name, spec, namespace, logger, **kwargs):
             ssh_keys="\n".join(ssh_keys),
             mounts=base64.b64encode(yaml.safe_dump(mounts).encode()).decode(),
             reload_signal=reload_signal,
+            reload_cmd=reload_cmd,
             post_mount_pod_cmd=post_mount_pod_cmd,
             kind=kind,
         ),
@@ -75,9 +78,13 @@ def update_mounts(name, spec, namespace, logger, **kwargs):
     """This handler will idempotently update the volume mounts."""
     del kwargs
     logger.info("Will idempotently update volume mounts.")
-    for manifest, mounted, mount_path, sub_path in iter_mounts_and_manifests(
-        namespace, spec["mounts"]
-    ):
+    for (
+        manifest,
+        mounted,
+        mount_path,
+        sub_path,
+        entrypoints,
+    ) in iter_mounts_and_manifests(namespace, spec["mounts"]):
         m_kind, m_name = manifest["kind"], manifest["metadata"]["name"]
 
         if spec["mountsEnabled"] and mounted:
@@ -112,10 +119,12 @@ def update_mounts(name, spec, namespace, logger, **kwargs):
                 mount_path=mount_path,
                 sub_path=sub_path,
             )
+            update_entrypoints(manifest=manifest, entrypoints=entrypoints)
             kubectl_apply(namespace=namespace, manifest=manifest, logger=logger)
         else:
             if spec.get("mode") == "modify":
                 remove_mount(manifest=manifest, volume_name=name)
+                restore_entrypoints(manifest=manifest, entrypoints=entrypoints)
                 logger.info("Idempotently unmounting volume to %s:%s", m_kind, m_name)
                 kubectl_apply(namespace=namespace, manifest=manifest, logger=logger)
             elif kubectl_get(namespace=namespace, kind=m_kind, labels={"devenv": name}):
@@ -142,7 +151,7 @@ def update_mounts(name, spec, namespace, logger, **kwargs):
 def cleanup_mounts(name, spec, namespace, logger, **kwargs):
     del kwargs
     logger.info("Clean up all volume mounts because dev env is being deleted.")
-    for manifest, _, _, _ in iter_mounts_and_manifests(namespace, spec["mounts"]):
+    for manifest, _, _, _, _ in iter_mounts_and_manifests(namespace, spec["mounts"]):
         remove_mount(manifest=manifest, volume_name=name)
         kubectl_apply(namespace=namespace, manifest=manifest, logger=logger)
 
@@ -213,7 +222,7 @@ def kubectl_get(namespace: str, kind: str, labels: dict[str, str]) -> list[dict]
 def iter_mounts_and_manifests(namespace, mounts):
     for mount in mounts:
         assert isinstance(mount, dict), repr(mount)
-        for attr in ("kind", "labels", "mountPath", "mounted"):
+        for attr in ("kind", "labels", "mountPath", "mounted", "entrypoints"):
             assert attr in mount, mount
         if mount["kind"].lower() != "deployment":
             raise NotImplementedError("Only deployments are supported.")
@@ -225,6 +234,7 @@ def iter_mounts_and_manifests(namespace, mounts):
                 mount["mounted"],
                 mount["mountPath"],
                 mount.get("subPath", ""),
+                mount["entrypoints"],
             )
 
 
@@ -281,3 +291,23 @@ def remove_mount(*, manifest: dict, volume_name: str) -> None:
         for i, mount in reversed(list(enumerate(mounts))):
             if mount["name"] == volume_name:
                 mounts.pop(i)
+
+
+def update_entrypoints(*, manifest: dict, entrypoints: dict) -> None:
+    for container in manifest["spec"]["template"]["spec"]["containers"]:
+        entrypoint = entrypoints.get(container["name"])
+        if entrypoint is None:
+            continue
+        cmd = shlex.split(entrypoint)
+        container["command"] = [cmd[0]]
+        container["args"] = cmd[1:]
+
+
+def restore_entrypoints(*, manifest: dict, entrypoints: dict) -> None:
+    for container in manifest["spec"]["template"]["spec"]["containers"]:
+        entrypoint = entrypoints.get(container["name"])
+        if entrypoint is None:
+            continue
+        if container.get("command"):
+            del container["command"]
+            del container["args"]

operator/templates/resource.yaml

@@ -42,6 +42,8 @@ spec:
           value: {name}
         - name: reload_signal
           value: {reload_signal}
+        - name: reload_cmd
+          value: {reload_cmd}
         - name: post_mount_pod_cmd
           value: {post_mount_pod_cmd}
         - name: namespace