feat: GCP integration GOS, GAR and GKE (#112)

This CL introduced GCP support with the following resources: - GKE clusters - Object storage buckets - Artifact registry repositories - Service Accounts - Networks Ticket: ENG-1610
Qovery · Feb 2, 2024 · f47b37a · f47b37a
1 parent 3909e5a
commit f47b37a
Show file tree

Hide file tree

Showing 25 changed files with 1,090 additions and 856 deletions.
diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml
@@ -16,7 +16,7 @@ jobs:
         name: Set up Go
         uses: actions/setup-go@main
         with:
-          go-version: 1.18
+          go-version: 1.21
       -
         name: Run golangci-lint
         uses: golangci/golangci-lint-action@v3

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -45,7 +45,7 @@ jobs:
         name: Set up Go
         uses: actions/setup-go@main
         with:
-          go-version: 1.18
+          go-version: 1.21
       -
         name: Run golangci-lint
         uses: golangci/golangci-lint-action@v2

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -24,7 +24,7 @@ permissions:
 env:
   # Go language version to use for building. This value should also be updated
   # in the release workflow if changed.
-  GO_VERSION: 1.18
+  GO_VERSION: 1.21
   TASK_VERSION: 3.x
 
 jobs:

diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.20.2-bullseye as build
+FROM golang:1.21.6-bullseye as build
 
 ADD . /pleco
 WORKDIR /pleco

diff --git a/README.md b/README.md
@@ -70,8 +70,13 @@ Check out our Blog announcement of Pleco: https://www.qovery.com/blog/announceme
   - [X] S3 Buckets
   - [X] Droplet firewalls
   - [X] Unused VPCs
+- [X] GCP
+  - [X] Cloud Storage Buckets
+  - [X] Artifact Registry Repositories
+  - [X] Kubernetes clusters
+  - [X] Networks // via JSON tags in resource description because resource has no support for tags
+  - [X] Service accounts // via JSON tags in resource description because resource has no support for tags
 - [ ] AZURE
-- [ ] GCP
 
 ---
 ## Installation
@@ -102,6 +107,11 @@ $ export DO_SPACES_KEY=<your_do_api_key_for_spaces>
 $ export DO_SPACES_SECRET=<your_do_api_secret_for_spaces>
 $ export DO_VOLUME_TIMEOUT=<delay_before_detached_volume_deletion_in_hours_since_creation> # default is 2 hours
 ```
+
+#### For GCP
+```bash
+$ export GOOGLE_APPLICATION_CREDENTIALS=<path_to_your_credentials_json_file>
+```
 ---
 ## Basic command
 
@@ -223,3 +233,47 @@ Here are some of the resources you can check:
 ```bash
 pleco start do --level debug -i 240 -a nyc3 -e -r -s -l -b -f -v -y
 ```
+
+### GCP options
+#### Region selector
+When pleco's look for expired resources, it will do it by [gcp_regions](https://cloud.google.com/compute/docs/regions-zones?hl=en).
+
+You can set zone(s) with:
+```bash
+--gcp-regions, -a <region(s)>
+```
+
+For example:
+```bash
+-a europe-west9
+```
+
+#### Resources Selector
+When pleco is running you have to specify which resources expiration will be checked.
+
+Here are some of the resources you can check:
+```bash
+--enable-cluster # Enable cluster watch
+--enable-object-storage # Enable object storage watch
+--enable-artifact-registry # Enable artifact registry watch
+--enable-network # Enable network watch
+--enable-iam # Enable IAM watch (service accounts)
+```
+
+#### Example
+```bash
+pleco start
+  gcp
+  --level
+  debug
+  -i
+  240
+  --enable-object-storage
+  --enable-artifact-registry
+  --enable-cluster
+  --enable-network
+  --enable-iam
+  --gcp-regions
+  europe-west9
+  --disable-dry-run
+```
diff --git a/charts/pleco/templates/deployment.yaml b/charts/pleco/templates/deployment.yaml
@@ -137,29 +137,51 @@ spec:
             {{ end }}
             {{- end }}
 
-#            Digital Ocean features
-            {{ if eq .Values.cloudProvider "do" }}
-            {{ if .Values.doFeatures.doRegions }}
-            - --do-regions
-            - "{{ join "," .Values.doFeatures.doRegions }}"
+#            GCP features
+            {{ if eq .Values.cloudProvider "gcp" }}
+            {{ if .Values.gcpFeatures.gcpRegions }}
+            - --gcp-regions
+            - "{{ join "," .Values.gcpFeatures.gcpRegions }}"
             {{ end }}
-            {{ if eq .Values.doFeatures.cluster true }}
+            {{ if eq .Values.gcpFeatures.cluster true }}
             - --enable-cluster
             {{ end }}
-            {{ if eq .Values.doFeatures.lb true }}
+            {{ if eq .Values.gcpFeatures.network true }}
+            - --enable-network
+            {{ end }}
+            {{ if eq .Values.gcpFeatures.iam true }}
+            - --enable-iam
+            {{ end }}
+            {{ if eq .Values.gcpFeatures.objectStorage true }}
+            - --enable-object-storage
+            {{ end }}
+            {{ if eq .Values.gcpFeatures.objectStorage true }}
+            - --enable-artifact-registry
+            {{ end }}
+            {{- end }}
+            #            Scaleway features
+            {{ if eq .Values.cloudProvider "scaleway" }}
+            {{ if .Values.scwFeatures.scwZones }}
+            - --scw-zones
+            - "{{ join "," .Values.scwFeatures.scwZones }}"
+            {{ end }}
+            {{ if eq .Values.scwFeatures.cr true}}
+            - --enable-cr
+            {{ end }}
+            {{ if eq .Values.scwFeatures.cluster true }}
+            - --enable-cluster
+            {{ end }}
+            {{ if eq .Values.scwFeatures.lb true }}
             - --enable-lb
             {{ end }}
-            {{ if eq .Values.doFeatures.db true }}
+            {{ if eq .Values.scwFeatures.db true }}
             - --enable-db
             {{ end }}
-            {{ if eq .Values.doFeatures.volume true }}
+            {{ if eq .Values.scwFeatures.volume true }}
             - --enable-volume
             {{ end }}
-            {{ if eq .Values.doFeatures.firewall true }}
-            - --enable-firewall
-            {{ end }}
-            {{ if eq .Values.doFeatures.vpc true }}
-            - --enable-vpc
+            {{ if eq .Values.scwFeatures.sg true }}
+            - --enable-sg
             {{ end }}
             {{- end }}
           env:

diff --git a/charts/pleco/values-gcp.yaml b/charts/pleco/values-gcp.yaml
@@ -0,0 +1,66 @@
+cloudProvider: "gcp"
+
+environmentVariables:
+  LOG_LEVEL: "debug"
+
+enabledFeatures:
+  disableDryRun: true
+  checkInterval: 120
+  disableTTLCheck: false
+  kubernetes: "in"
+  s3: true
+
+gcpFeatures:
+  gcpRegions:
+  - asia-east1,
+  - asia-east2,
+  - asia-northeast1,
+  - asia-northeast2,
+  - asia-northeast3,
+  - asia-south1,
+  - asia-south2,
+  - asia-southeast1,
+  - asia-southeast2,
+  - australia-southeast1,
+  - australia-southeast2,
+  - europe-central2,
+  - europe-north1,
+  - europe-southwest1,
+  - europe-west1,
+  - europe-west10,
+  - europe-west12,
+  - europe-west2,
+  - europe-west3,
+  - europe-west4,
+  - europe-west6,
+  - europe-west8,
+  - europe-west9,
+  - me-central1,
+  - me-central2,
+  - me-west1,
+  - northamerica-northeast1,
+  - northamerica-northeast2,
+  - southamerica-east1,
+  - southamerica-west1,
+  - us-central1,
+  - us-east1,
+  - us-east4,
+  - us-east5,
+  - us-south1,
+  - us-west1,
+  - us-west2,
+  - us-west3,
+  - us-west4
+  cluster: true
+  network: true
+  iam: true
+  artifactRegistry: true
+  objectStorage: true
+
+resources:
+  limits:
+    cpu: 250m
+    memory: 512Mi
+  requests:
+    cpu: 250m
+    memory: 512Mi
diff --git a/charts/pleco/values.yaml b/charts/pleco/values.yaml
@@ -23,6 +23,8 @@ environmentVariables:
   #  DO_SPACES_KEY: ""
   #  DO_SPACES_SECRET: ""
   #  DO_VOLUME_TIMEOUT: ""
+  # GCP
+  #  GOOGLE_APPLICATION_CREDENTIALS_JSON: ""
 
 
 enabledFeatures:
@@ -64,11 +66,20 @@ scwFeatures:
 doFeatures:
   doRegions: [ ]
   #  - nyc3
-  cluster: flase
+  cluster: false
   lb: false
   db: false
   volume: false
 
+gcpFeatures:
+  gcpRegions: []
+  # - europe-west9
+  cluster: false
+  network: false
+  iam: false
+  objectStorage: false
+  artifactRegistry: false
+
 imagePullSecrets: []
 nameOverride: ""
 fullnameOverride: ""

diff --git a/cmd/start.go b/cmd/start.go
@@ -27,7 +27,7 @@ var startCmd = &cobra.Command{
 		log.Infof("Starting Pleco %s", GetCurrentVersion())
 
 		if len(args) < 1 {
-			log.Errorf("The provider is mandatory (aws | scaleway | do)")
+			log.Errorf("The provider is mandatory (aws | scaleway | do | gcp)")
 			os.Exit(1)
 		}