[IBCDPE-938] Deploy Signoz (OTEL Visualization) to kubernetes cluster

.gitignore

@@ -1,4 +1,5 @@
 *.tfstate*
 .terraform
 terraform.tfvars
-settings.json
+settings.json
+temporary_files*

deployments/main.tf

@@ -37,15 +37,7 @@ module "dpe-sandbox-spacelift-development" {
   cluster_name = "dpe-k8-sandbox"
   vpc_name     = "dpe-sandbox"
 
-  vpc_cidr_block = "10.51.0.0/16"
-  # public_subnet_cidrs  = ["10.51.1.0/24", "10.51.2.0/24", "10.51.3.0/24"]
-  # private_subnet_cidrs = ["10.51.4.0/24", "10.51.5.0/24", "10.51.6.0/24"]
-  # azs                  = ["us-east-1a", "us-east-1b", "us-east-1c"]
-  # For now, we are only using one public and one private subnet. This is due to how
-  # EBS can only be mounted to a single AZ. We will need to revisit this if we want to
-  # allow usage of EFS ($$$$), or add some kind of EBS volume replication.
-  # Note: EKS requires at least two subnets in different AZs. However, we are only using
-  # a single subnet for node deployment.
+  vpc_cidr_block       = "10.51.0.0/16"
   public_subnet_cidrs  = ["10.51.1.0/24", "10.51.2.0/24"]
   private_subnet_cidrs = ["10.51.4.0/24", "10.51.5.0/24"]
   azs                  = ["us-east-1a", "us-east-1b"]
@@ -75,15 +67,7 @@ module "dpe-sandbox-spacelift-production" {
   cluster_name = "dpe-k8"
   vpc_name     = "dpe-k8"
 
-  vpc_cidr_block = "10.52.0.0/16"
-  # public_subnet_cidrs  = ["10.52.1.0/24", "10.52.2.0/24", "10.52.3.0/24"]
-  # private_subnet_cidrs = ["10.52.4.0/24", "10.52.5.0/24", "10.52.6.0/24"]
-  # azs                  = ["us-east-1a", "us-east-1b", "us-east-1c"]
-  # For now, we are only using one public and one private subnet. This is due to how
-  # EBS can only be mounted to a single AZ. We will need to revisit this if we want to
-  # allow usage of EFS ($$$$), or add some kind of EBS volume replication.
-  # Note: EKS requires at least two subnets in different AZs. However, we are only using
-  # a single subnet for node deployment.
+  vpc_cidr_block       = "10.52.0.0/16"
   public_subnet_cidrs  = ["10.52.1.0/24", "10.52.2.0/24"]
   private_subnet_cidrs = ["10.52.4.0/24", "10.52.5.0/24"]
   azs                  = ["us-east-1a", "us-east-1b"]

deployments/stacks/dpe-k8s-deployments/main.tf

@@ -6,7 +6,7 @@ module "sage-aws-eks-autoscaler" {
   vpc_id                 = var.vpc_id
   node_security_group_id = var.node_security_group_id
   spotinst_account       = var.spotinst_account
-  single_az              = true
+  single_az              = false
   desired_capacity       = 3
 }
 
@@ -21,8 +21,9 @@ module "sage-aws-eks-addons" {
 
 module "argo-cd" {
   depends_on = [module.sage-aws-eks-autoscaler]
-  source     = "spacelift.io/sagebionetworks/argo-cd/aws"
-  version    = "0.3.1"
+  # source     = "spacelift.io/sagebionetworks/argo-cd/aws"
+  # version    = "0.3.1"
+  source = "../../../modules/argo-cd"
 }
 
 module "victoria-metrics" {
@@ -66,10 +67,69 @@ module "postgres-cloud-native-database" {
   depends_on           = [module.postgres-cloud-native-operator, module.airflow, module.argo-cd]
   source               = "spacelift.io/sagebionetworks/postgres-cloud-native-database/aws"
   version              = "0.5.0"
-  auto_deploy          = true
-  auto_prune           = true
+  auto_deploy          = var.auto_deploy
+  auto_prune           = var.auto_prune
   git_revision         = var.git_revision
   namespace            = "airflow"
   argo_deployment_name = "airflow-postgres-cloud-native"
 }
 
+
+module "signoz" {
+  depends_on = [module.argo-cd]
+  # source               = "spacelift.io/sagebionetworks/postgres-cloud-native-database/aws"
+  # version              = "0.5.0"
+  source               = "../../../modules/signoz"
+  auto_deploy          = var.auto_deploy
+  auto_prune           = var.auto_prune
+  git_revision         = var.git_revision
+  namespace            = "signoz"
+  argo_deployment_name = "signoz"
+}
+
+module "envoy-gateway" {
+  # TODO: This is temporary until we are ready to deploy the ingress controller: https://sagebionetworks.jira.com/browse/IBCDPE-1095
+  count      = 0
+  depends_on = [module.argo-cd]
+  # source               = "spacelift.io/sagebionetworks/postgres-cloud-native-database/aws"
+  # version              = "0.5.0"
+  source               = "../../../modules/envoy-gateway"
+  auto_deploy          = var.auto_deploy
+  auto_prune           = var.auto_prune
+  git_revision         = var.git_revision
+  namespace            = "envoy-gateway"
+  argo_deployment_name = "envoy-gateway"
+}
+
+module "cert-manager" {
+  # TODO: This is temporary until we are ready to deploy the ingress controller: https://sagebionetworks.jira.com/browse/IBCDPE-1095
+  count      = 0
+  depends_on = [module.argo-cd]
+  # source               = "spacelift.io/sagebionetworks/postgres-cloud-native-database/aws"
+  # version              = "0.5.0"
+  source               = "../../../modules/cert-manager"
+  auto_deploy          = var.auto_deploy
+  auto_prune           = var.auto_prune
+  git_revision         = var.git_revision
+  namespace            = "cert-manager"
+  argo_deployment_name = "cert-manager"
+}
+
+module "cluster-ingress" {
+  # TODO: This is temporary until we are ready to deploy the ingress controller: https://sagebionetworks.jira.com/browse/IBCDPE-1095
+  count      = 0
+  depends_on = [module.argo-cd]
+  # source               = "spacelift.io/sagebionetworks/postgres-cloud-native-database/aws"
+  # version              = "0.5.0"
+  source               = "../../../modules/cluster-ingress"
+  auto_deploy          = var.auto_deploy
+  auto_prune           = var.auto_prune
+  git_revision         = var.git_revision
+  namespace            = "envoy-gateway"
+  argo_deployment_name = "cluster-ingress"
+
+  # To determine more elegant ways to fill in these values, for example, if we have
+  # a pre-defined DNS name for the cluster (https://sagebionetworks.jira.com/browse/IT-3931)
+  ssl_hostname        = "unknown-to-fill-in"
+  cluster_issuer_name = "selfsigned"
+}

modules/apache-airflow/templates/values.yaml

@@ -108,7 +108,9 @@ images:
     pullPolicy: IfNotPresent
 
 # Select certain nodes for airflow pods.
-nodeSelector: {}
+nodeSelector: {
+  failure-domain.beta.kubernetes.io/zone: us-east-1a
+}
 affinity: {}
 tolerations: []
 topologySpreadConstraints: []
@@ -467,7 +469,7 @@ kerberos:
 # Airflow Worker Config
 workers:
   # Number of airflow celery workers in StatefulSet
-  replicas: 2
+  replicas: 1
   # Max number of old replicasets to retain
   revisionHistoryLimit: ~
 
@@ -636,7 +638,9 @@ workers:
   extraVolumeMounts: []
 
   # Select certain nodes for airflow worker pods.
-  nodeSelector: {}
+  nodeSelector: {
+    spotinst.io/node-lifecycle: "od"
+  }
   runtimeClassName: ~
   priorityClassName: ~
   affinity:
@@ -721,7 +725,7 @@ scheduler:
     command: ~
   # Airflow 2.0 allows users to run multiple schedulers,
   # However this feature is only recommended for MySQL 8+ and Postgres
-  replicas: 2
+  replicas: 1
   # Max number of old replicasets to retain
   revisionHistoryLimit: ~
 
@@ -806,7 +810,9 @@ scheduler:
   extraVolumeMounts: []
 
   # Select certain nodes for airflow scheduler pods.
-  nodeSelector: {}
+  nodeSelector: {
+    spotinst.io/node-lifecycle: "od"
+  }
   affinity:
   # default scheduler affinity is:
    podAntiAffinity:
@@ -1248,7 +1254,7 @@ webserver:
 triggerer:
   enabled: true
   # Number of airflow triggerers in the deployment
-  replicas: 2
+  replicas: 1
   # Max number of old replicasets to retain
   revisionHistoryLimit: ~
 
@@ -1348,7 +1354,9 @@ triggerer:
   extraVolumeMounts: []
 
   # Select certain nodes for airflow triggerer pods.
-  nodeSelector: {}
+  nodeSelector: {
+    spotinst.io/node-lifecycle: "od"
+  }
   affinity:
   # default triggerer affinity is:
    podAntiAffinity:
@@ -1942,7 +1950,9 @@ redis:
   safeToEvict: true
 
   # Select certain nodes for redis pods.
-  nodeSelector: {}
+  nodeSelector: {
+    spotinst.io/node-lifecycle: "od"
+  }
   affinity: {}
   tolerations: []
   topologySpreadConstraints: []

modules/argo-cd/templates/values.yaml

@@ -501,7 +501,14 @@ configs:
   # -- Repositories list to be used by applications
   ## Creates a secret for each key/value specified below to create repositories
   ## Note: the last example in the list would use a repository credential template, configured under "configs.credentialTemplates".
-  repositories: {}
+  repositories:
+    docker-registry:
+      url: registry-1.docker.io
+      # username: "docker"
+      # password: ""
+      name: docker-registry
+      enableOCI: "true"
+      type: "helm"
     # istio-helm-repo:
     #   url: https://storage.googleapis.com/istio-prerelease/daily-build/master-latest-daily/charts
     #   name: istio.io

modules/cert-manager/README.md

@@ -0,0 +1,13 @@
+# Purpose
+This module is used to deploy the cert-manager helm chart. cert-manager is responsible
+for creating SSL certs to use within the cluster.
+
+Resources:
+
+- <https://cert-manager.io/docs/installation/helm/>
+
+## Relation to envoy-gateway
+The envoy-gateway is responsible for handling ingress for the kubernetes cluster. 
+cert-manager has an a integration to watch for changes to `kind: Gateway` resources to
+determine when to provision SSL certs. This integration is in the `values.yaml` file
+of this directory under `kind: ControllerConfiguration`.

modules/cert-manager/main.tf

@@ -0,0 +1,38 @@
+resource "kubernetes_namespace" "cert-manager" {
+  metadata {
+    name = var.namespace
+  }
+}
+
+resource "kubectl_manifest" "cert-manager" {
+  depends_on = [kubernetes_namespace.cert-manager]
+
+  yaml_body = <<YAML
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: cert-manager
+  namespace: argocd
+spec:
+  project: default
+  %{if var.auto_deploy}
+  syncPolicy:
+    automated:
+      prune: ${var.auto_prune}
+  %{endif}
+  sources:
+  - repoURL: 'https://charts.jetstack.io'
+    chart: cert-manager
+    targetRevision: v1.15.1
+    helm:
+      releaseName: cert-manager
+      valueFiles:
+      - $values/modules/cert-manager/templates/values.yaml
+  - repoURL: 'https://github.com/Sage-Bionetworks-Workflows/eks-stack.git'
+    targetRevision: ${var.git_revision}
+    ref: values
+  destination:
+    server: 'https://kubernetes.default.svc'
+    namespace: ${var.namespace}
+YAML
+}