diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 371bcb3..a5e8999 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,11 +16,11 @@ include: # Defines .base job template. - project: frontier/cloud-hosted-beta/eoaas-deployment-pipeline file: gitlab-ci-templates/base.gitlab-ci.yml - ref: usr/dr/gmtxm + ref: main # Defines .crane and .crane-copy-multi-image templates. - project: frontier/cloud-hosted-beta/eoaas-deployment-pipeline file: gitlab-ci-templates/crane.gitlab-ci.yml - ref: usr/dr/gmtxm + ref: main variables: # Variables with descriptions so that a nice form is generated in the Gitlab UI @@ -28,33 +28,36 @@ variables: DEVENV_NAME: description: Name of the DevEnv that will be created. value: "" - EO_ENV: - description: Name of the EO deployment that will be targeted + APP_ENV: + description: Name of the app deployment that will be targeted. value: "" NAMESPACE: - description: Namespace where to deploy the DevEnv and look for the target services - value: hzp + description: Namespace where to deploy the DevEnv and look for the target services. + value: default SELECTORS: - description: Labels of target EO services to develop for + description: Labels of target EO services to develop for. value: app=api-v2 MOUNT_PATH: - description: The path in the target pod were the code will be mounted at + description: The path in the target pod were the code will be mounted at. value: "/mist" MODE: - description: Choose if target pod should be modified or cloned + description: Choose if target pod should be modified or cloned. value: "modify" options: ["modify", "clone"] PORT: - description: The port of the target pod that should be exposed when using the clone mode + description: The port of the target pod that should be exposed when using the clone mode. value: "8080" RELOAD_SIGNAL: - description: The Unix signal to send to the target pod to reload the code + description: The Unix signal to send to the target pod to reload the code. value: "HUP" options: ["TERM", "HUP"] + RELOAD_CMD: + description: The command to run to target pods when reloading the code. + value: "" POST_MOUNT_POD_CMD: - description: The command to run after the target pod mounts the code + description: The command to run after the target pod mounts the code. EXCLUDED: - description: Excluded paths + description: Excluded paths. value: '[".git", ".vscode", "__pycache__", ".pyc", "/landing", "/portal", "/jenkins", "/settings", "/ui", "/tests", "/docker/nginx/static"]' # Remaining variables that will not be rendered in the Gitlab manual pipeline form. @@ -111,16 +114,16 @@ deploy operator: GIT_STRATEGY: fetch stage: deploy operator rules: - - if: $EO_ENV == "" + - if: $APP_ENV == "" when: manual - when: always script: - |- - if [ -z "$EO_ENV" ]; then + if [ -z "$APP_ENV" ]; then echo "Deploying operator on host cluster" else - echo "Deploying operator inside $EO_ENV vcluster" - vcluster connect $EO_ENV + echo "Deploying operator inside $APP_ENV vcluster" + vcluster connect $APP_ENV sleep 5 fi kubectl cluster-info @@ -144,23 +147,28 @@ deploy devenv: needs: - deploy operator rules: - - if: $EO_ENV == "" + - if: $APP_ENV == "" when: manual - if: $DEVENV_NAME == "" when: manual - when: always script: - |- - echo "Deploying DevEnv" - vcluster connect $EO_ENV - echo "Connected to vcluster" - kubectl cluster-info + if [ -z "$APP_ENV" ]; then + echo "Deploying DevEnv on host cluster" + else + echo "Deploying DevEnv" + vcluster connect $APP_ENV + echo "Connected to vcluster" + kubectl cluster-info + fi echo "Excluded:" $EXCLUDED echo "Mount path:" $MOUNT_PATH echo "Selectors:" $SELECTORS echo "Mode:" $MODE echo "Port:" $PORT echo "Reload signal:" $RELOAD_SIGNAL + echo "Reload command:" $RELOAD_CMD echo "Post mount command:" $POST_MOUNT_POD_CMD curl https://gitlab.dell.com/$GITLAB_USER_LOGIN.keys > user.keys export GITHUB_USER_LOGIN=$(echo $GITLAB_USER_LOGIN|tr '_' '-') @@ -170,6 +178,7 @@ deploy devenv: export MOUNTS="" export KIND="deployment" export GROUP_NAME="eoaas-development" + export ENTRYPOINTS="{}" for SELECTOR in $SELECTORS do export LABELS=$(echo $SELECTOR |sed -r 's/=/:\ /g'|sed -r 's/,/\n/g'|sed -r 's/^/ /g'|sed '1i\\') diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 056b28f..6d95081 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,7 +13,7 @@ repos: - --branch=main - --pattern=^rel/ - id: check-added-large-files - args: ["--maxkb=50"] + args: ["--maxkb=360"] - id: check-merge-conflict - id: check-case-conflict - id: fix-byte-order-marker diff --git a/Operator.png b/Operator.png new file mode 100644 index 0000000..31ed37f Binary files /dev/null and b/Operator.png differ diff --git a/README.md b/README.md index 320ecb5..83e6ddc 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # Remote Development Operator -The Remote Development Operator, is a Kubernetes operator and CRD for deploying remote development environments. It has been developed within Dell ISG Edge, to facilitate the development efforts of __[Dell NativeEdge](https://www.dell.com/en-us/dt/solutions/edge-computing/edge-platform.htm)__, the edge operations platform. The Remote Development Operator can be leveraged to facilitate the development of any software project that can be deployed on Kubernetes. +The Remote Development Operator, is a Kubernetes operator and Custom Resource Definition (CRD) for deploying remote development environments. It has been developed within Dell ISG Edge, to facilitate the development efforts of __[Dell NativeEdge](https://www.dell.com/en-us/dt/solutions/edge-computing/edge-platform.htm)__, the edge operations platform. The Remote Development Operator can be leveraged to facilitate the development of any software project that can be deployed on Kubernetes. + + ## Table of Contents @@ -12,11 +14,20 @@ The Remote Development Operator, is a Kubernetes operator and CRD for deploying - [License](#license) # Features -The CRD, defines a new Kubernetes resource named __DevEnv__. Each __DevEnv__ can target one or more k8s deployments or statefulsets using label selectors. For each __DevEnv__, the operator creates a DNS record, a PVC for storing the code or binaries, and starts an SSH server, configured to accept the specified SSH keypairs. Each __DevEnv__, provides a command that can be used to configure the end user's IDE (e.g. VSCode). This configuration will synchronize over SSH the local code that's stored on the developer's device to the PVC that's attached to the __DevEnv__ pod. After the first sync is complete, the __DevEnv__ gets enabled and mounts the code PVC to the target pods. +We define a new Kubernetes custom resource named __DevEnv__. Each __DevEnv__ can target one or more k8s deployments or statefulsets using label selectors. For each __DevEnv__, the operator creates a DNS record, a PVC for storing the code or binaries, and starts an SSH server, configured to accept the specified SSH keypairs. Each __DevEnv__, provides a command that can be used to configure the end user's IDE (e.g. VSCode). This configuration will synchronize over SSH the local code that's stored on the developer's device to the PVC that's attached to the __DevEnv__ pod. After the first sync is complete, the __DevEnv__ gets enabled and mounts the code PVC to the target pods. There are two supported modes of operation. -- In `modify` mode the operator will edit the definition of the target deployments or statefulsets, mounting the code PVC on the target path, which will override the original code that's burned into the image. -- In `clone` mode the operator will launch a new deployment or statefulset, with the PVC mounted on the target path, while leaving the original deployment or statefulset untouched. A new service and ingress is also generated, to expose a port of the cloned deployment over HTTPS. +### Modify mode + + + +In `modify` mode the operator will edit the definition of the target deployments or statefulsets, mounting the code PVC on the target path, which will override the original code that's burned into the image. + +## Clone mode + + + +In `clone` mode the operator will launch a new deployment or statefulset, with the PVC mounted on the target path, while leaving the original deployment or statefulset untouched. A new service and ingress is also generated, to expose a port of the cloned deployment over HTTPS. When working on a large application that consists of numerous microservices, the `clone` mode allows multiple developers to work in parallel, leveraging the same deployed application. There may still be cases were the cloned devenv may affect the original application, e.g. when working on DB schema changes. Developers that leverage the `clone` mode should coordinate with the peers to ensure they're not breaking their development environments. When in doubt, `modify` is the safest option, however it mostly requires a dedicated application deployment for each developer. @@ -50,6 +61,7 @@ Once the operator is installed, you can start creating DevEnvs. Copy examples/de | mounts | A list of mounts | | excludedPaths | The repository paths to be excluded from syncing. | | reloadSignal | The UNIX signal that will force the target resource to reload its code. Can be `TERM` or `HUP`. | +| reloadCmd | A command to run on the pod of the target resource when reloading the code. | | postMountPodCmd | A command to run on the pod of the target resource after mounting the code PVC. | || diff --git a/clone-mode.svg b/clone-mode.svg new file mode 100644 index 0000000..e00f6f5 --- /dev/null +++ b/clone-mode.svg @@ -0,0 +1 @@ +Appsvc4svc3svc2svc1……Remote clusterDevEnvSSH SERVERRSYNCCode PVCsvc1cloneHTTPSIDEGIT diff --git a/devenv/.bashrc b/devenv/.bashrc index 77a58ff..1b8d92a 100644 --- a/devenv/.bashrc +++ b/devenv/.bashrc @@ -81,13 +81,5 @@ if [ -f ~/.bash_aliases ]; then . ~/.bash_aliases fi -# enable programmable completion features (you don't need to enable -# this, if it's already enabled in /etc/bash.bashrc and /etc/profile -# sources /etc/bash.bashrc). -if ! shopt -oq posix; then - if [ -f /usr/share/bash-completion/bash_completion ]; then - . /usr/share/bash-completion/bash_completion - elif [ -f /etc/bash_completion ]; then - . /etc/bash_completion - fi -fi +# enable programmable completion features +source /etc/bash_completion diff --git a/devenv/Dockerfile b/devenv/Dockerfile index cf8cc96..80414cf 100644 --- a/devenv/Dockerfile +++ b/devenv/Dockerfile @@ -4,7 +4,7 @@ RUN useradd -ms /bin/bash $USER ENV HOME /home/$USER USER root # RUN adduser $USER -RUN apt-get update && apt-get install jq curl fuse sudo sed apt-utils vim openssh-server gzip git rsync -y +RUN apt-get update && apt-get install jq curl fuse sudo sed apt-utils vim openssh-server gzip git rsync bash-completion -y RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/`curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt`/bin/linux/amd64/kubectl && \ chmod +x ./kubectl && \ mv ./kubectl /usr/local/bin/kubectl && \ diff --git a/devenv/scripts/reload.sh b/devenv/scripts/reload.sh index 9290a0c..cc1e145 100755 --- a/devenv/scripts/reload.sh +++ b/devenv/scripts/reload.sh @@ -1,9 +1,35 @@ #!/bin/bash # Send reload signal to target pods. -for pod in `kubectl get po -lapp=api-v2 -o name` +export pods=$(python -d <IDEGITAppsvc4svc3svc2svc1……Remote clusterDevEnvSSH SERVERRSYNCCode PVC diff --git a/operator/crd.yaml b/operator/crd.yaml index 744ad64..ad66b7a 100644 --- a/operator/crd.yaml +++ b/operator/crd.yaml @@ -107,6 +107,11 @@ spec: description: The labels that have been assigned to target Deployment or Statefulset. additionalProperties: type: string + entrypoints: + type: object + description: Override the entrypoint of specific containers in the target Deployment or Statefulset when mounting the code volume. + additionalProperties: + type: string mounted: type: boolean description: Whether the volume should actually be mounted. @@ -118,6 +123,10 @@ spec: enum: - TERM - HUP + reloadCmd: + type: string + description: Command to be executed to target pods when reloading the code. + default: '' postMountPodCmd: type: string description: Command to be executed to target pods after mounting the code volume. diff --git a/operator/op.py b/operator/op.py index ecbcf33..e4cd64b 100644 --- a/operator/op.py +++ b/operator/op.py @@ -3,6 +3,7 @@ import base64 import functools import os +import shlex import subprocess from copy import deepcopy @@ -33,6 +34,7 @@ def create_update_dev_env(name, spec, namespace, logger, **kwargs): excluded_paths = spec["excludedPaths"] mounts = spec["mounts"] reload_signal = spec["reloadSignal"] + reload_cmd = spec["reloadCmd"] post_mount_pod_cmd = spec["postMountPodCmd"] # Interpolate all templates and apply idempotently using kubectl apply -f - @@ -53,6 +55,7 @@ def create_update_dev_env(name, spec, namespace, logger, **kwargs): ssh_keys="\n".join(ssh_keys), mounts=base64.b64encode(yaml.safe_dump(mounts).encode()).decode(), reload_signal=reload_signal, + reload_cmd=reload_cmd, post_mount_pod_cmd=post_mount_pod_cmd, kind=kind, ), @@ -75,9 +78,13 @@ def update_mounts(name, spec, namespace, logger, **kwargs): """This handler will idempotently update the volume mounts.""" del kwargs logger.info("Will idempotently update volume mounts.") - for manifest, mounted, mount_path, sub_path in iter_mounts_and_manifests( - namespace, spec["mounts"] - ): + for ( + manifest, + mounted, + mount_path, + sub_path, + entrypoints, + ) in iter_mounts_and_manifests(namespace, spec["mounts"]): m_kind, m_name = manifest["kind"], manifest["metadata"]["name"] if spec["mountsEnabled"] and mounted: @@ -112,10 +119,12 @@ def update_mounts(name, spec, namespace, logger, **kwargs): mount_path=mount_path, sub_path=sub_path, ) + update_entrypoints(manifest=manifest, entrypoints=entrypoints) kubectl_apply(namespace=namespace, manifest=manifest, logger=logger) else: if spec.get("mode") == "modify": remove_mount(manifest=manifest, volume_name=name) + restore_entrypoints(manifest=manifest, entrypoints=entrypoints) logger.info("Idempotently unmounting volume to %s:%s", m_kind, m_name) kubectl_apply(namespace=namespace, manifest=manifest, logger=logger) elif kubectl_get(namespace=namespace, kind=m_kind, labels={"devenv": name}): @@ -142,7 +151,7 @@ def update_mounts(name, spec, namespace, logger, **kwargs): def cleanup_mounts(name, spec, namespace, logger, **kwargs): del kwargs logger.info("Clean up all volume mounts because dev env is being deleted.") - for manifest, _, _, _ in iter_mounts_and_manifests(namespace, spec["mounts"]): + for manifest, _, _, _, _ in iter_mounts_and_manifests(namespace, spec["mounts"]): remove_mount(manifest=manifest, volume_name=name) kubectl_apply(namespace=namespace, manifest=manifest, logger=logger) @@ -213,7 +222,7 @@ def kubectl_get(namespace: str, kind: str, labels: dict[str, str]) -> list[dict] def iter_mounts_and_manifests(namespace, mounts): for mount in mounts: assert isinstance(mount, dict), repr(mount) - for attr in ("kind", "labels", "mountPath", "mounted"): + for attr in ("kind", "labels", "mountPath", "mounted", "entrypoints"): assert attr in mount, mount if mount["kind"].lower() != "deployment": raise NotImplementedError("Only deployments are supported.") @@ -225,6 +234,7 @@ def iter_mounts_and_manifests(namespace, mounts): mount["mounted"], mount["mountPath"], mount.get("subPath", ""), + mount["entrypoints"], ) @@ -281,3 +291,23 @@ def remove_mount(*, manifest: dict, volume_name: str) -> None: for i, mount in reversed(list(enumerate(mounts))): if mount["name"] == volume_name: mounts.pop(i) + + +def update_entrypoints(*, manifest: dict, entrypoints: dict) -> None: + for container in manifest["spec"]["template"]["spec"]["containers"]: + entrypoint = entrypoints.get(container["name"]) + if entrypoint is None: + continue + cmd = shlex.split(entrypoint) + container["command"] = [cmd[0]] + container["args"] = cmd[1:] + + +def restore_entrypoints(*, manifest: dict, entrypoints: dict) -> None: + for container in manifest["spec"]["template"]["spec"]["containers"]: + entrypoint = entrypoints.get(container["name"]) + if entrypoint is None: + continue + if container.get("command"): + del container["command"] + del container["args"] diff --git a/operator/templates/resource.yaml b/operator/templates/resource.yaml index b8ac48d..6b5fb9f 100644 --- a/operator/templates/resource.yaml +++ b/operator/templates/resource.yaml @@ -42,6 +42,8 @@ spec: value: {name} - name: reload_signal value: {reload_signal} + - name: reload_cmd + value: {reload_cmd} - name: post_mount_pod_cmd value: {post_mount_pod_cmd} - name: namespace