[SCHEMATIC-138] SigNoz cold storage and backups

README.md

@@ -172,7 +172,7 @@ allow us to review for any security advisories.
 
 ### Deploying an application to the kubernetes cluster
 Deployment of applications to the kubernetes cluster is handled through the combination
-of terraform (.tf) scripts, spacelift (CICD tool), and ArgoCd (Declarative definitions 
+of terraform (.tf) scripts, spacelift (CICD tool), and ArgoCd or Flux CD (Declarative definitions 
 for applications).
 
 To start of the deployment journey the first step is to create a new terraform module

deployments/spacelift/dpe-k8s/main.tf

@@ -45,6 +45,7 @@ locals {
     pod_to_node_dns_sg_id               = "TF_VAR_pod_to_node_dns_sg_id"
     smtp_user                           = "TF_VAR_smtp_user"
     smtp_password                       = "TF_VAR_smtp_password"
+    cluster_oidc_provider_arn           = "TF_VAR_cluster_oidc_provider_arn"
   }
 }
 
@@ -178,31 +179,6 @@ resource "spacelift_stack_dependency_reference" "cluster-name" {
 #   stack_id  = spacelift_stack.k8s-stack.id
 # }
 
-resource "spacelift_stack_destructor" "k8s-stack-deployments-destructor" {
-  depends_on = [
-    spacelift_stack.k8s-stack,
-    spacelift_aws_integration_attachment.k8s-deployments-aws-integration-attachment,
-    spacelift_context_attachment.k8s-kubeconfig-hooks,
-    spacelift_stack_dependency_reference.cluster-name,
-    spacelift_stack_dependency_reference.region-name,
-    spacelift_environment_variable.k8s-stack-deployments-environment-variables
-  ]
-
-  stack_id = spacelift_stack.k8s-stack-deployments.id
-}
-
-resource "spacelift_stack_destructor" "k8s-stack-destructor" {
-  depends_on = [
-    spacelift_aws_integration_attachment.k8s-aws-integration-attachment,
-    spacelift_context_attachment.k8s-kubeconfig-hooks,
-    spacelift_stack_dependency_reference.cluster-name,
-    spacelift_stack_dependency_reference.region-name,
-    spacelift_environment_variable.k8s-stack-environment-variables
-  ]
-
-  stack_id = spacelift_stack.k8s-stack.id
-}
-
 resource "spacelift_aws_integration_attachment" "k8s-aws-integration-attachment" {
   integration_id = var.aws_integration_id
   stack_id       = spacelift_stack.k8s-stack.id
@@ -244,12 +220,6 @@ resource "spacelift_stack" "auth0" {
   ]
 }
 
-resource "spacelift_stack_destructor" "auth0-stack-destructor" {
-  count    = var.deploy_auth0 ? 1 : 0
-  stack_id = spacelift_stack.auth0[0].id
-}
-
-
 resource "spacelift_environment_variable" "auth0-stack-environment-variables" {
   depends_on = [
     spacelift_stack.auth0

deployments/stacks/dpe-k8s-deployments/main.tf

@@ -1,3 +1,6 @@
+locals {
+  git_revision = var.git_revision
+}
 module "sage-aws-eks-autoscaler" {
   source                 = "spacelift.io/sagebionetworks/sage-aws-eks-autoscaler/aws"
   version                = "0.9.0"
@@ -26,13 +29,19 @@ module "argo-cd" {
   source = "../../../modules/argo-cd"
 }
 
+module "flux-cd" {
+  depends_on = [module.sage-aws-eks-autoscaler]
+  source     = "../../../modules/flux-cd"
+}
+
 module "victoria-metrics" {
-  depends_on   = [module.argo-cd]
-  source       = "spacelift.io/sagebionetworks/victoria-metrics/aws"
-  version      = "0.4.8"
+  depends_on = [module.argo-cd]
+  # source       = "spacelift.io/sagebionetworks/victoria-metrics/aws"
+  # version      = "0.4.8"
+  source       = "../../../modules/victoria-metrics"
   auto_deploy  = var.auto_deploy
   auto_prune   = var.auto_prune
-  git_revision = var.git_revision
+  git_revision = local.git_revision
 }
 
 module "trivy-operator" {
@@ -41,7 +50,7 @@ module "trivy-operator" {
   version      = "0.3.2"
   auto_deploy  = var.auto_deploy
   auto_prune   = var.auto_prune
-  git_revision = var.git_revision
+  git_revision = local.git_revision
 }
 
 module "airflow" {
@@ -50,7 +59,7 @@ module "airflow" {
   version      = "0.4.0"
   auto_deploy  = var.auto_deploy
   auto_prune   = var.auto_prune
-  git_revision = var.git_revision
+  git_revision = local.git_revision
   namespace    = "airflow"
 }
 
@@ -60,7 +69,7 @@ module "postgres-cloud-native-operator" {
   version      = "0.4.0"
   auto_deploy  = var.auto_deploy
   auto_prune   = var.auto_prune
-  git_revision = var.git_revision
+  git_revision = local.git_revision
 }
 
 module "postgres-cloud-native-database" {
@@ -69,30 +78,40 @@ module "postgres-cloud-native-database" {
   version              = "0.5.0"
   auto_deploy          = var.auto_deploy
   auto_prune           = var.auto_prune
-  git_revision         = var.git_revision
+  git_revision         = local.git_revision
   namespace            = "airflow"
   argo_deployment_name = "airflow-postgres-cloud-native"
 }
 
+module "clickhouse-backup-bucket" {
+  source                    = "../../../modules/s3-bucket"
+  bucket_name               = "clickhouse-backup-${var.aws_account_id}-${var.cluster_name}"
+  enable_versioning         = false
+  aws_account_id            = var.aws_account_id
+  cluster_name              = var.cluster_name
+  cluster_oidc_provider_arn = var.cluster_oidc_provider_arn
+}
 
 module "signoz" {
   depends_on = [module.argo-cd]
   # source               = "spacelift.io/sagebionetworks/postgres-cloud-native-database/aws"
   # version              = "0.5.0"
-  source               = "../../../modules/signoz"
-  auto_deploy          = var.auto_deploy
-  auto_prune           = var.auto_prune
-  git_revision         = var.git_revision
-  namespace            = "signoz"
-  argo_deployment_name = "signoz"
-  enable_otel_ingress  = var.enable_otel_ingress && var.enable_cluster_ingress
-  gateway_namespace    = "envoy-gateway"
-  cluster_name         = var.cluster_name
-  auth0_jwks_uri       = var.auth0_jwks_uri
-  smtp_password        = var.smtp_password
-  smtp_user            = var.smtp_user
-  smtp_from            = var.smtp_from
-  auth0_identifier     = var.auth0_identifier
+  source                = "../../../modules/signoz"
+  auto_deploy           = var.auto_deploy
+  auto_prune            = var.auto_prune
+  git_revision          = local.git_revision
+  namespace             = "signoz"
+  argo_deployment_name  = "signoz"
+  enable_otel_ingress   = var.enable_otel_ingress && var.enable_cluster_ingress
+  gateway_namespace     = "envoy-gateway"
+  cluster_name          = var.cluster_name
+  auth0_jwks_uri        = var.auth0_jwks_uri
+  smtp_password         = var.smtp_password
+  smtp_user             = var.smtp_user
+  smtp_from             = var.smtp_from
+  auth0_identifier      = var.auth0_identifier
+  s3_backup_bucket_name = module.clickhouse-backup-bucket.bucket_name
+  s3_access_role_arn    = module.clickhouse-backup-bucket.access_role_arn
 }
 
 module "envoy-gateway" {
@@ -103,7 +122,7 @@ module "envoy-gateway" {
   source               = "../../../modules/envoy-gateway"
   auto_deploy          = var.auto_deploy
   auto_prune           = var.auto_prune
-  git_revision         = var.git_revision
+  git_revision         = local.git_revision
   namespace            = "envoy-gateway"
   argo_deployment_name = "envoy-gateway"
   cluster_issuer_name  = "lets-encrypt-prod"
@@ -118,7 +137,7 @@ module "cert-manager" {
   source               = "../../../modules/cert-manager"
   auto_deploy          = var.auto_deploy
   auto_prune           = var.auto_prune
-  git_revision         = var.git_revision
+  git_revision         = local.git_revision
   namespace            = "cert-manager"
   argo_deployment_name = "cert-manager"
 }

deployments/stacks/dpe-k8s-deployments/variables.tf

@@ -40,6 +40,11 @@ variable "cluster_name" {
   type        = string
 }
 
+variable "cluster_oidc_provider_arn" {
+  description = "EKS cluster ARN for the oidc provider"
+  type        = string
+}
+
 variable "spotinst_account" {
   description = "Spot.io account"
   type        = string

deployments/stacks/dpe-k8s/outputs.tf

@@ -38,11 +38,15 @@ output "cluster_name" {
   value = module.sage-aws-eks.cluster_name
 }
 
+output "cluster_oidc_provider_arn" {
+  value = module.sage-aws-eks.cluster_oidc_provider_arn
+}
+
 output "smtp_user" {
-  value = length(module.sage-aws-ses) > 0 ? module.sage-aws-ses[0].smtp_user : null
+  value = length(module.sage-aws-ses) > 0 ? module.sage-aws-ses[0].smtp_user : ""
 }
 
 output "smtp_password" {
   sensitive = true
-  value     = length(module.sage-aws-ses) > 0 ? module.sage-aws-ses[0].smtp_password : null
+  value     = length(module.sage-aws-ses) > 0 ? module.sage-aws-ses[0].smtp_password : ""
 }

modules/flux-cd/README.md

@@ -0,0 +1,56 @@
+# Purpose
+This module is used to deploy the `Flux CD` [helm chart](https://fluxcd-community.github.io/helm-charts)  to the cluster. [`Flux CD`](https://fluxcd.io/) is a GitOps tool used to manage the application lifecycle on a Kubernetes cluster. It was originally deployed because unlike `Argo CD`, it supports the use of `postRenderers` which are used to apply any additional changes to the application after it has been deployed, and which were needed to be used to deploy the `clickhouse-backup` sidecar container to the `signoz` helm release. We do not plan to move all existing applications to `fluxcd` at this time, but it is available and preferred to be used for any new applications added to the cluster.
+
+## What resources are being deployed through this module
+In addition to a `helm_release` which deploys the `fluxcd` helm chart, this module also creates a `capacitor` resource which is used as the frontend for `fluxcd`.
+
+## Accessing the Flux CD UI
+To access the `Flux CD` UI, you only need to port-forward the `capacitor` pod and access it from your browser.
+
+# Deploying an application with Flux CD
+To deploy an application with `Flux CD`, will need to create a `HelmRepository` resource which points to the helm chart you want to deploy. In that resource definition, you will set the `apiVersion` to `source.toolkit.fluxcd.io/v1` and the `kind` to `HelmRepository`. For example (code from the `signoz` module):
+
+```
+resource "kubectl_manifest" "signoz-helm-repo" {
+  depends_on = [kubernetes_namespace.signoz]
+
+  yaml_body = <<YAML
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: signoz
+  namespace: ${var.namespace}
+spec:
+  interval: 24h
+  url: https://charts.signoz.io
+YAML
+}
+```
+
+In the `Deployment` or `HelmRelease` resource, you will need to add a similar configuration, for example (again from the `signoz` module):
+```
+resource "kubectl_manifest" "signoz-helm-release" {
+  depends_on = [kubernetes_namespace.signoz]
+
+  yaml_body = <<YAML
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: signoz
+  namespace: ${var.namespace}
+spec:
+  interval: 10m
+  chart:
+    spec:
+      chart: signoz
+      version: '0.55.1'
+      sourceRef:
+        kind: HelmRepository
+        name: signoz
+        namespace: ${var.namespace}
+      interval: 10m
+      reconcileStrategy: Revision
+...
+YAML
+}
+```

modules/flux-cd/main.tf

@@ -0,0 +1,56 @@
+resource "kubernetes_namespace" "flux-system" {
+  metadata {
+    name = "flux-system"
+  }
+}
+
+resource "helm_release" "fluxcd" {
+  name       = "flux2"
+  repository = "https://fluxcd-community.github.io/helm-charts"
+  chart      = "flux2"
+  namespace  = "flux-system"
+  version    = "2.14.0"
+  depends_on = [kubernetes_namespace.flux-system]
+
+  values = [templatefile("${path.module}/templates/values.yaml", {})]
+}
+
+resource "kubectl_manifest" "capacitor" {
+  depends_on = [helm_release.fluxcd]
+
+  yaml_body = <<YAML
+apiVersion: source.toolkit.fluxcd.io/v1beta2
+kind: OCIRepository
+metadata:
+  name: capacitor
+  namespace: flux-system
+spec:
+  interval: 12h
+  url: oci://ghcr.io/gimlet-io/capacitor-manifests
+  ref:
+    semver: ">=0.1.0"
+YAML
+}
+
+resource "kubectl_manifest" "capacitor-kustomization" {
+  depends_on = [helm_release.fluxcd]
+
+  yaml_body = <<YAML
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: capacitor
+  namespace: flux-system
+spec:
+  targetNamespace: flux-system
+  interval: 1h
+  retryInterval: 2m
+  timeout: 5m
+  wait: true
+  prune: true
+  path: "./"
+  sourceRef:
+    kind: OCIRepository
+    name: capacitor
+YAML
+}