From 697e04cd50909b7b893266b3c3fee606c10ec26a Mon Sep 17 00:00:00 2001 From: Ricardo Campos Date: Thu, 25 Aug 2022 07:57:03 -0300 Subject: [PATCH] Fix/merge main flow (#20) * fix: ci workflows for pr close and merge main * fix: new job names * fix: remove database deployment for now * fix: wrong yaml syntax * fix: adjust if condition to run on pull requests * ci: fix codeql into pull requests flow * fix: removed database deploy config files * fix: enable codeql to run in every commit --- .../deploy.backend-with-database.yml | 159 +++++++++++++++ .github/openshift/deploy.backend.yml | 18 -- .github/workflows/merge-main.yml | 11 +- .github/workflows/pr-close.yml | 5 +- .github/workflows/pr-open.yml | 186 +++++++++++------- docker-compose-with-database.yml | 40 ++++ docker-compose.yml | 28 --- 7 files changed, 322 insertions(+), 125 deletions(-) create mode 100644 .github/openshift/deploy.backend-with-database.yml create mode 100644 docker-compose-with-database.yml diff --git a/.github/openshift/deploy.backend-with-database.yml b/.github/openshift/deploy.backend-with-database.yml new file mode 100644 index 0000000..dcb7780 --- /dev/null +++ b/.github/openshift/deploy.backend-with-database.yml @@ -0,0 +1,159 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: ${NAME} + annotations: + description: "NR Backend Starting API" + tags: "nrbestapi" + iconClass: icon-java +labels: + app: ${NAME}-${ZONE} + app.kubernetes.io/part-of: ${NAME}-${ZONE} +parameters: + - name: NAME + description: Module name + value: nrbestapi + - name: COMPONENT + description: Component name + value: service-api + - name: ZONE + description: Deployment zone, e.g. pr-### or prod + required: true + - name: IMAGE_TAG + description: Image tag to use + value: latest + - name: DOMAIN + value: apps.silver.devops.gov.bc.ca + - name: CPU_REQUEST + value: 100m + - name: MEMORY_REQUEST + value: 0.5Gi + - name: REGISTRY + description: Container registry to import from (internal is image-registry.openshift-image-registry.svc:5000) + value: ghcr.io + - name: PROMOTE + description: Image (namespace/name:tag) to promote/import + value: bcgov/nr-backend-starting-api:prod-service-api +objects: + - apiVersion: v1 + kind: ImageStream + metadata: + labels: + app: ${NAME}-${ZONE} + name: ${NAME}-${ZONE}-${COMPONENT} + spec: + lookupPolicy: + local: false + tags: + - name: ${IMAGE_TAG} + from: + kind: DockerImage + name: ${REGISTRY}/${PROMOTE} + referencePolicy: + type: Local + - apiVersion: v1 + kind: DeploymentConfig + metadata: + labels: + app: ${NAME}-${ZONE} + name: ${NAME}-${ZONE}-${COMPONENT} + spec: + replicas: 1 + triggers: + - type: ConfigChange + - type: ImageChange + imageChangeParams: + automatic: true + containerNames: + - ${NAME} + from: + kind: ImageStreamTag + name: ${NAME}-${ZONE}-${COMPONENT}:${IMAGE_TAG} + selector: + deploymentconfig: ${NAME}-${ZONE}-${COMPONENT} + strategy: + type: Rolling + template: + metadata: + labels: + app: ${NAME}-${ZONE} + deploymentconfig: ${NAME}-${ZONE}-${COMPONENT} + spec: + containers: + - image: ${NAME}-${ZONE}-${COMPONENT}:${IMAGE_TAG} + imagePullPolicy: Always + name: ${NAME} + env: + - name: POSTGRESQL_HOST + value: ${NAME}-${ZONE}-database + - name: POSTGRESQL_DATABASE + valueFrom: + secretKeyRef: + name: ${NAME}-${ZONE}-database + key: database-name + - name: POSTGRESQL_PASSWORD + valueFrom: + secretKeyRef: + name: ${NAME}-${ZONE}-database + key: database-password + - name: POSTGRESQL_USER + valueFrom: + secretKeyRef: + name: ${NAME}-${ZONE}-database + key: database-user + ports: + - containerPort: 8090 + protocol: TCP + resources: + requests: + cpu: ${CPU_REQUEST} + memory: ${MEMORY_REQUEST} + readinessProbe: + httpGet: + path: /check + port: 8090 + scheme: HTTP + initialDelaySeconds: 120 + periodSeconds: 15 + timeoutSeconds: 10 + livenessProbe: + successThreshold: 1 + failureThreshold: 3 + httpGet: + path: /check + port: 8090 + scheme: HTTP + initialDelaySeconds: 120 + periodSeconds: 30 + timeoutSeconds: 10 + - apiVersion: v1 + kind: Service + metadata: + labels: + app: ${NAME}-${ZONE} + name: ${NAME}-${ZONE}-${COMPONENT} + spec: + ports: + - name: 8090-tcp + protocol: TCP + port: 80 + targetPort: 8090 + selector: + deploymentconfig: ${NAME}-${ZONE}-${COMPONENT} + - apiVersion: route.openshift.io/v1 + kind: Route + metadata: + labels: + app: ${NAME}-${ZONE} + name: ${NAME}-${ZONE}-${COMPONENT} + spec: + host: ${NAME}-${ZONE}-${COMPONENT}.${DOMAIN} + port: + targetPort: 8090-tcp + to: + kind: Service + name: ${NAME}-${ZONE}-${COMPONENT} + weight: 100 + tls: + termination: edge + insecureEdgeTerminationPolicy: Redirect diff --git a/.github/openshift/deploy.backend.yml b/.github/openshift/deploy.backend.yml index dcb7780..1421ce5 100644 --- a/.github/openshift/deploy.backend.yml +++ b/.github/openshift/deploy.backend.yml @@ -83,24 +83,6 @@ objects: - image: ${NAME}-${ZONE}-${COMPONENT}:${IMAGE_TAG} imagePullPolicy: Always name: ${NAME} - env: - - name: POSTGRESQL_HOST - value: ${NAME}-${ZONE}-database - - name: POSTGRESQL_DATABASE - valueFrom: - secretKeyRef: - name: ${NAME}-${ZONE}-database - key: database-name - - name: POSTGRESQL_PASSWORD - valueFrom: - secretKeyRef: - name: ${NAME}-${ZONE}-database - key: database-password - - name: POSTGRESQL_USER - valueFrom: - secretKeyRef: - name: ${NAME}-${ZONE}-database - key: database-user ports: - containerPort: 8090 protocol: TCP diff --git a/.github/workflows/merge-main.yml b/.github/workflows/merge-main.yml index 2e8a96d..c64fe20 100644 --- a/.github/workflows/merge-main.yml +++ b/.github/workflows/merge-main.yml @@ -5,10 +5,7 @@ on: branches: - main paths-ignore: - - ".**" - "**.md" - - "**.yml" - - "**.yaml" workflow_dispatch: env: @@ -157,12 +154,12 @@ jobs: oc login --token=${{ secrets.OC_TOKEN }} --server=${{ secrets.OC_SERVER }} oc project ${{ secrets.OC_NAMESPACE }} # Do not replace database; 'oc create' kicks up an error if objects already exist - oc process -f .github/openshift/deploy.database.yml -p ZONE=${{ env.ZONE }} | oc create -f - || true + #oc process -f .github/openshift/deploy.database.yml -p ZONE=${{ env.ZONE }} | oc create -f - || true # Process and apply deployment templates oc process -f .github/openshift/deploy.backend.yml -p ZONE=${{ env.ZONE }} \ -p PROMOTE=${{ github.repository }}:${{ env.ZONE }}-service-api | oc apply -f - # Follow any active rollouts (see deploymentconfigs) - oc rollout status dc/${{ env.NAME }}-${{ env.ZONE }}-database -w + #oc rollout status dc/${{ env.NAME }}-${{ env.ZONE }}-database -w oc rollout status dc/${{ env.NAME }}-${{ env.ZONE }}-service-api -w # Remove completed build runs, build pods and deployment pods for p in $(oc get po | grep "Completed" | awk '{print $1}') @@ -296,12 +293,12 @@ jobs: fi # Process and apply deployment templates - oc process -f .github/openshift/deploy.database.yml -p ZONE=${{ env.ZONE }} | oc apply -f - + #oc process -f .github/openshift/deploy.database.yml -p ZONE=${{ env.ZONE }} | oc apply -f - oc process -f .github/openshift/deploy.backend.yml -p ZONE=${{ env.ZONE }} \ -p PROMOTE=${{ github.repository }}:${{ env.PREV }}-service-api | oc apply -f - # Follow any active rollouts (see deploymentconfigs) - oc rollout status dc/${{ env.NAME }}-${{ env.ZONE }}-database -w + #oc rollout status dc/${{ env.NAME }}-${{ env.ZONE }}-database -w oc rollout status dc/${{ env.NAME }}-${{ env.ZONE }}-service-api -w # Remove completed build runs, build pods and deployment pods diff --git a/.github/workflows/pr-close.yml b/.github/workflows/pr-close.yml index af19057..aab2759 100644 --- a/.github/workflows/pr-close.yml +++ b/.github/workflows/pr-close.yml @@ -1,14 +1,11 @@ -name: Pull Request Close +name: PR Close on: pull_request: types: - closed paths-ignore: - - ".**" - "**.md" - - "**.yml" - - "**.yaml" env: REGISTRY: ghcr.io diff --git a/.github/workflows/pr-open.yml b/.github/workflows/pr-open.yml index 236fc60..4adc1e6 100644 --- a/.github/workflows/pr-open.yml +++ b/.github/workflows/pr-open.yml @@ -1,7 +1,14 @@ -name: Pull Request Open +name: Commits and PR Open on: pull_request: + push: + branches: + - '*' + - '*/*' + - '**' + tags-ignore: + - '**' env: REGISTRY: ghcr.io @@ -12,65 +19,9 @@ concurrency: cancel-in-progress: true jobs: - pr-validation: - name: Pull Request Label Validation - runs-on: ubuntu-latest - permissions: - contents: read - pull-requests: write - steps: - - uses: actions/checkout@v3 - - - name: Pull request size and stability labels - uses: actions/labeler@v4 - continue-on-error: true - with: - repo-token: "${{ secrets.GITHUB_TOKEN }}" - - - name: Conventional Label - uses: bcoe/conventional-release-labels@v1 - continue-on-error: true - with: - token: ${{ secrets.GITHUB_TOKEN }} - ignored_types: '["chore","pr"]' - type_labels: '{"feat": "feature", "fix": "fix", "bug": "fix", "doc": "documentation", "ci": "ci", "chore": "chore", "breaking": "breaking", "BREAKING CHANGE": "breaking"}' - - - name: Checkout branch - uses: actions/checkout@v3 - with: - ref: refs/heads/${{ github.head_ref }} - - - name: Conventional Changelog Update - continue-on-error: true - uses: TriPSs/conventional-changelog-action@v3 - id: changelog - with: - github-token: ${{ github.token }} - output-file: 'CHANGELOG.md' - skip-version-file: 'true' - skip-commit: 'true' - git-push: 'false' - git-branch: refs/heads/${{ github.head_ref }} - - - name: Checkout pr - uses: actions/checkout@v3 - with: - ref: ${{ github.ref }} - - - name: Comment PR - continue-on-error: true - uses: thollander/actions-comment-pull-request@v1 - if: ${{ steps.changelog.outputs.skipped == 'false' }} - with: - message: | - # Current changelog - - ${{ steps.changelog.outputs.clean_changelog }} - comment_includes: '# Current changelog' - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - + # For every commit and pull request validate: - name: Code validation + name: Code validation (Commits and PR) runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -104,8 +55,9 @@ jobs: path: target/site/checkstyle.html retention-days: 5 - sonarcloud: - name: Test Code and Static Analysis + # For every commit and pull request + tests: + name: Code tests (Commits and PR) runs-on: ubuntu-latest needs: - validate @@ -175,8 +127,9 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + # For every commit and pull request security: - name: Security checks + name: Security checks (Commits and PR) runs-on: ubuntu-latest permissions: security-events: write @@ -218,15 +171,110 @@ jobs: with: args: --severity-threshold=high --sarif-file-output=snyk.sarif + # For every commit and pull request + codeql: + name: Semantic Code Analysis (Commits and PR) + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + needs: + - tests + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Initialize + uses: github/codeql-action/init@v2 + with: + debug: true + languages: java + + - name: Set up JDK 17 + uses: actions/setup-java@v2 + with: + java-version: '17' + distribution: 'adopt' + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 + + # Only for Pull Requests + pr-validation: + name: Label Validation (Only PR) + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' && github.event.pull_request.base.ref == 'main' + permissions: + contents: read + pull-requests: write + steps: + - uses: actions/checkout@v3 + + - name: Pull request size and stability labels + uses: actions/labeler@v4 + continue-on-error: true + with: + repo-token: "${{ secrets.GITHUB_TOKEN }}" + + - name: Conventional Label + uses: bcoe/conventional-release-labels@v1 + continue-on-error: true + with: + token: ${{ secrets.GITHUB_TOKEN }} + ignored_types: '["chore","pr"]' + type_labels: '{"feat": "feature", "fix": "fix", "bug": "fix", "doc": "documentation", "ci": "ci", "chore": "chore", "breaking": "breaking", "BREAKING CHANGE": "breaking"}' + + - name: Checkout branch + uses: actions/checkout@v3 + with: + ref: refs/heads/${{ github.head_ref }} + + - name: Conventional Changelog Update + continue-on-error: true + uses: TriPSs/conventional-changelog-action@v3 + id: changelog + with: + github-token: ${{ github.token }} + output-file: 'CHANGELOG.md' + skip-version-file: 'true' + skip-commit: 'true' + git-push: 'false' + git-branch: refs/heads/${{ github.head_ref }} + + - name: Checkout pr + uses: actions/checkout@v3 + with: + ref: ${{ github.ref }} + + - name: Comment PR + continue-on-error: true + uses: thollander/actions-comment-pull-request@v1 + if: ${{ steps.changelog.outputs.skipped == 'false' }} + with: + message: | + # Current changelog + + ${{ steps.changelog.outputs.clean_changelog }} + comment_includes: '# Current changelog' + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # Only for Pull Requests build-service-api: - name: Service API Image Build + name: Image Build (Only PR) env: COMPONENT: service-api ZONE: ${{ github.event.number }} runs-on: ubuntu-latest + if: github.event_name == 'pull_request' && github.event.pull_request.base.ref == 'main' needs: - security - - sonarcloud + - tests permissions: contents: read packages: write @@ -266,13 +314,15 @@ jobs: # Clean previous image oc delete is ${{ env.NAME }}-${{ env.ZONE }}-${{ env.COMPONENT }} || true + # Only for Pull Requests deploy-dev: - name: DEV Deployment + name: DEV Deployment (Only PR) needs: - security - - sonarcloud + - tests - build-service-api runs-on: ubuntu-latest + if: github.event_name == 'pull_request' && github.event.pull_request.base.ref == 'main' timeout-minutes: 15 env: ZONE: ${{ github.event.number }} @@ -285,7 +335,7 @@ jobs: oc project ${{ secrets.OC_NAMESPACE }} # Database uses a default build - oc process -f .github/openshift/deploy.database.yml -p ZONE=${{ env.ZONE }} | oc apply -f - + # oc process -f .github/openshift/deploy.database.yml -p ZONE=${{ env.ZONE }} | oc apply -f - # Clean previous image, if rebuilding if [ ${{ needs.build-service-api.outputs.build == 'true' }} ] @@ -300,7 +350,7 @@ jobs: -p PROMOTE=${{ github.repository }}:${{ env.ZONE }}-service-api | oc apply -f - # Follow any active rollouts (see deploymentconfigs) - oc rollout status dc/${{ env.NAME }}-${{ env.ZONE }}-database -w + #oc rollout status dc/${{ env.NAME }}-${{ env.ZONE }}-database -w oc rollout status dc/${{ env.NAME }}-${{ env.ZONE }}-service-api -w - name: DEV Deployment update uses: mshick/add-pr-comment@v1 diff --git a/docker-compose-with-database.yml b/docker-compose-with-database.yml new file mode 100644 index 0000000..48e8c05 --- /dev/null +++ b/docker-compose-with-database.yml @@ -0,0 +1,40 @@ +version: '3.9' + +services: + database: + container_name: database + image: postgres:12 + environment: + POSTGRES_USER: default + POSTGRES_PASSWORD: default + POSTGRES_DB: default + hostname: database + restart: always + healthcheck: + test: ["CMD-SHELL", "pg_isready -U default"] + interval: 10s + timeout: 5s + retries: 5 + volumes: + - /pgdata + + backend: + depends_on: + database: + condition: service_healthy + container_name: service-api + image: service-api:latest + environment: + NODE_ENV: development + POSTGRESQL_HOST: database + POSTGRESQL_USER: default + POSTGRESQL_PASSWORD: default + POSTGRESQL_DATABASE: default + hostname: service-api + links: + - database + ports: + - "8090:8090" + build: + context: . + dockerfile: Dockerfile diff --git a/docker-compose.yml b/docker-compose.yml index 48e8c05..674866e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,38 +1,10 @@ version: '3.9' services: - database: - container_name: database - image: postgres:12 - environment: - POSTGRES_USER: default - POSTGRES_PASSWORD: default - POSTGRES_DB: default - hostname: database - restart: always - healthcheck: - test: ["CMD-SHELL", "pg_isready -U default"] - interval: 10s - timeout: 5s - retries: 5 - volumes: - - /pgdata - backend: - depends_on: - database: - condition: service_healthy container_name: service-api image: service-api:latest - environment: - NODE_ENV: development - POSTGRESQL_HOST: database - POSTGRESQL_USER: default - POSTGRESQL_PASSWORD: default - POSTGRESQL_DATABASE: default hostname: service-api - links: - - database ports: - "8090:8090" build: