feat(apps/prod/tekton/configs/tasks): add oras push task (#777)

Signed-off-by: wuhuizuo <wuhuizuo@126.com>

---------

Signed-off-by: wuhuizuo <wuhuizuo@126.com>

apps/prod/tekton/configs/pipelines/push-build-package.yaml

@@ -24,13 +24,13 @@ spec:
 
         May be it is an idea to judge it by git-url, but it maybe 
         not a generic way for forked repositories with custom names.
+    - name: binary-builder-iamge
+      description: The image used to build binaries.
   workspaces:
-    - name: dockerconfig
-      description: Includes a docker `config.json`
     - name: source
       description: The workspace where the git repo will be cloned.
-    - name: gen-build-scripts
-      description: The build scripts will be generated in this workspace.
+    - name: dockerconfig
+      description: Includes a docker `config.json`
     - name: git-credentials
       description: secret contains ssh private key in `id_rsa` key.
       optional: true
@@ -72,9 +72,9 @@ spec:
       workspaces:
         - name: source
           workspace: source
-    - name: generate-building-scripts-for-tarballs
+    - name: build-binaries
       taskRef:
-        name: generate-building-scripts-for-tarballs
+        name: pingcap-build-binaries
       runAfter:
         - get-release-ver
       # TODO: currently matrix feature is not support in v0.32.x, we need upgrade the K8S cluster and then upgrade Tekton.
@@ -101,23 +101,25 @@ spec:
           value: $(params.git-ref)
         - name: git-sha
           value: $(params.git-revision)
+        - name: builder-image
+          # value: ghcr.io/pingcap-qe/ci/release-build-base:v20231029-b8b8d34-go1.21
+          value: $(params.binary-builder-iamge)
+        - name: release-dir
+          value: build
+        - name: push
+          value: 'true'
       workspaces:
-        - name: output
-          workspace: gen-build-scripts
-    - name: generate-building-scripts-for-images
+        - name: source
+          workspace: source
+        - name: dockerconfig
+          workspace: dockerconfig
+    - name: build-images
       taskRef:
-        name: generate-building-scripts-for-images
+        # TODO: can we skaffold to make it simpler?
+        name: pingcap-build-images
       runAfter:
-        - get-release-ver
+        - build-binaries
       # TODO: currently matrix feature is not support in v0.32.x, we need upgrade the K8S cluster and then upgrade Tekton.
-      # matrix:
-      #   params:
-      #     - name: os
-      #       value: [linux, darwin]
-      #     - name: arch
-      #       value: [amd64, arm64]
-      #     - name: profile
-      #       value: [release, debug]
       params:
         - name: os
           value: linux
@@ -126,59 +128,19 @@ spec:
         - name: profile
           value: release
         - name: component
-          value: "$(params.component)"
+          value: "$(params.component)"     
         - name: version
-          value: "$(tasks.get-release-ver.results.version)"
+          value: "$(tasks.get-release-ver.results.version)"           
         - name: git-ref
           value: $(params.git-ref)
         - name: git-sha
           value: $(params.git-revision)
-      workspaces:
-        - name: output
-          workspace: gen-build-scripts
-    - name: run-building-scripts-for-tarballs
-      taskRef:
-        # TODO: is there a sulution to push for multi platforms.
-        name: run-building-scripts-for-tarballs
-      runAfter:
-        - generate-building-scripts-for-tarballs
-      # TODO: currently matrix feature is not support in v0.32.x, we need upgrade the K8S cluster and then upgrade Tekton.
-      params:
-        - name: release-dir
-          value: build
-        - name: run-script-file
-          value: build-package-artifacts.sh
-        - name: need-build
-          value: "true"
-        - name: build-image
-          # value: ghcr.io/pingcap-qe/ci/release-build-base:v20231029-b8b8d34-go1.21
-          value: ghcr.io/pingcap-qe/ci/release-build-base:v20231107-4086b32-go1.21_linux_amd64
-      workspaces:
-        - name: source
-          workspace: source
-        - name: build-scripts
-          workspace: gen-build-scripts
-    - name: run-building-scripts-for-images
-      taskRef:
-        # TODO: can we skaffold to make it simpler?
-        name: run-building-scripts-for-images
-      runAfter:
-        - generate-building-scripts-for-images
-        - run-building-scripts-for-tarballs
-      # TODO: currently matrix feature is not support in v0.32.x, we need upgrade the K8S cluster and then upgrade Tekton.
-      params:
         - name: release-dir
           value: build
-        - name: run-script-file
-          value: build-package-images.sh
-        - name: need-build
+        - name: build
           value: "false"
-        - name: build-image
-          value: gcr.io/kaniko-project/executor:debug
       workspaces:
         - name: source
           workspace: source
-        - name: build-scripts
-          workspace: gen-build-scripts
         - name: dockerconfig
           workspace: dockerconfig

apps/prod/tekton/configs/tasks/kustomization.yaml

@@ -16,5 +16,7 @@ resources:
   - ko-test-infra.yaml
   - kubernetes-actions.yaml
   - multi-arch-image-push.yaml
+  - pingcap-build-binaries.yaml
+  - pingcap-build-images.yaml
   - run-build-binaries-scripts.yaml
   - run-build-images-scripts.yaml

apps/prod/tekton/configs/tasks/pingcap-build-binaries.yaml

@@ -0,0 +1,73 @@
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: pingcap-build-binaries
+  labels:
+    app.kubernetes.io/version: "0.1"
+  annotations:
+    tekton.dev/platforms: "linux/amd64"
+spec:
+  description: >-
+    Build binaries for the component.
+  workspaces:
+    - name: source
+    - name: dockerconfig
+      description: Includes a docker `config.json`
+      mountPath: /root/.docker
+      optional: true
+  results:
+    - description: The package artifacts
+      name: package-artifacts
+      type: string
+  params:
+    - name: component
+    - name: os
+      default: linux
+    - name: arch
+      default: amd64
+    - name: version
+      type: string
+      description: the version to releasing.
+    - name: profile
+      type: string
+      default: release
+      description: >
+        supports: 'release' or 'debug'.
+    - name: git-ref
+    - name: git-sha
+    - name: builder-image
+      default: ghcr.io/pingcap-qe/ci/release-utils:v20231107-4086b32
+    - name: release-dir
+      default: build
+    - name: push
+      type: string
+      default: 'false'
+  steps:
+    - name: generate-build-script
+      image: ghcr.io/pingcap-qe/ci/release-utils:v20231107-4086b32
+      script: |
+        git clone --depth=1 --branch=main https://github.com/PingCAP-QE/artifacts.git /workspace/artifacts
+
+        /workspace/artifacts/packages/scripts/gen-package-artifacts-with-config.sh \
+          $(params.component) \
+          $(params.os) \
+          $(params.arch) \
+          $(params.version) \
+          $(params.profile) \
+          $(params.git-ref) \
+          $(params.git-sha) \
+          /workspace/artifacts/packages/packages.yaml.tmpl \
+          /workspace/build-package-artifacts.sh
+
+        cat /workspace/build-package-artifacts.sh
+    - name: build
+      image: "$(params.builder-image)"
+      workingDir: $(workspaces.source.path)
+      script: |
+        /workspace/build-package-artifacts.sh -b -a -w "$(params.release-dir)"
+    - name: publish
+      image: ghcr.io/pingcap-qe/ci/release-utils:v20231107-4086b32
+      workingDir: $(workspaces.source.path)
+      script: |
+        oras version
+        /workspace/build-package-artifacts.sh -p -w "$(params.release-dir)"

apps/prod/tekton/configs/tasks/pingcap-build-images.yaml

@@ -0,0 +1,69 @@
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: pingcap-build-images
+  labels:
+    app.kubernetes.io/version: "0.1"
+  annotations:
+    tekton.dev/platforms: "linux/amd64"
+spec:
+  description: >-
+    This task builds images for pingcap components.
+  workspaces:
+    - name: source
+    - name: dockerconfig
+      description: Includes a docker `config.json`
+      mountPath: /kaniko/.docker      
+      optional: true
+  results:
+    - description: Digest of the image just built.
+      name: IMAGE_DIGEST
+    - description: URL of the image just built.
+      name: IMAGE_URL
+  params:
+    - name: component
+    - name: os
+      default: linux
+    - name: arch
+      default: amd64
+    - name: version
+      type: string
+      description: the version to releasing.
+    - name: profile
+      type: string
+      default: release
+      description: >
+        supports: 'release' or 'debug'.
+    - name: git-ref
+    - name: git-sha
+    - name: release-dir
+      default: build
+    - name: build
+      default: 'false'
+      description: Does it need to build the binaries before packing the images?
+  steps:
+    - name: generate
+      image: ghcr.io/pingcap-qe/ci/release-utils:v20231107-4086b32
+      script: |
+        git clone --depth=1 --branch=main https://github.com/PingCAP-QE/artifacts.git /workspace/artifacts
+
+        /workspace/artifacts/packages/scripts/gen-package-images-with-config.sh \
+          $(params.component) \
+          $(params.os) \
+          $(params.arch) \
+          $(params.version) \
+          $(params.profile) \
+          $(params.git-ref) \
+          $(params.git-sha) \
+          /workspace/artifacts/packages/packages.yaml.tmpl \
+          /workspace/build-package-images.sh
+
+        cat /workspace/build-package-images.sh
+    - name: build-and-publish
+      image: gcr.io/kaniko-project/executor:debug
+      workingDir: $(workspaces.source.path)
+      env:
+        - name: KANIKO_EXECUTOR
+          value: /kaniko/executor
+      script: |
+        /workspace/build-package-images.sh "$(params.release-dir)" "$(params.tag)" ${KANIKO_EXECUTOR} "$(params.build)"

apps/prod/tekton/configs/tasks/run-build-binaries-scripts.yaml

@@ -19,6 +19,9 @@ spec:
       default: build-package-artifacts.sh
     - name: need-build
       default: 'true'
+    - name: push
+      type: string
+      default: 'false'
   steps:
     - name: build
       image: "$(params.build-image)"

apps/prod/tekton/configs/triggers/templates/tikv/pd/push.yaml

@@ -32,6 +32,17 @@ spec:
             value: $(tt.params.git-ref)
           - name: component
             value: pd
+          - name: binary-builder-iamge
+            value: ghcr.io/pingcap-qe/cd/builders/pd:v20231115-e1c4b43-go1.21
+        taskRunSpecs:
+          - pipelineTaskName: build-binaries
+            taskPodTemplate:
+              nodeSelector:
+                kubernetes.io/arch: amd64
+          - pipelineTaskName: build-images
+            taskPodTemplate:
+              nodeSelector:
+                kubernetes.io/arch: amd64
         workspaces:
           - name: dockerconfig
             secret:
@@ -45,17 +56,3 @@ spec:
                 resources:
                   requests:
                     storage: 2Gi
-          - name: gen-build-scripts
-            volumeClaimTemplate:
-              spec:
-                storageClassName: ceph-block
-                accessModes:
-                  - ReadWriteOnce
-                resources:
-                  requests:
-                    storage: 100Mi
-        taskRunSpecs:
-          - pipelineTaskName: run-building-scripts-for-tarballs
-            taskPodTemplate:
-              nodeSelector:
-                kubernetes.io/arch: amd64

apps/prod/tekton/configs/triggers/templates/tikv/tikv/push.yaml

@@ -32,6 +32,17 @@ spec:
             value: $(tt.params.git-ref)
           - name: component
             value: tikv
+          - name: binary-builder-iamge
+            value: ghcr.io/pingcap-qe/cd/builders/tikv:v20231116-e1c4b43
+        taskRunSpecs:
+          - pipelineTaskName: build-binaries
+            taskPodTemplate:
+              nodeSelector:
+                kubernetes.io/arch: amd64
+          - pipelineTaskName: build-images
+            taskPodTemplate:
+              nodeSelector:
+                kubernetes.io/arch: amd64
         workspaces:
           - name: dockerconfig
             secret:
@@ -44,18 +55,4 @@ spec:
                   - ReadWriteOnce
                 resources:
                   requests:
-                    storage: 10Gi
-          - name: gen-build-scripts
-            volumeClaimTemplate:
-              spec:
-                storageClassName: ceph-block
-                accessModes:
-                  - ReadWriteOnce
-                resources:
-                  requests:
-                    storage: 100Mi
-        taskRunSpecs:
-          - pipelineTaskName: run-building-scripts-for-tarballs
-            taskPodTemplate:
-              nodeSelector:
-                kubernetes.io/arch: amd64
+                    storage: 50Gi

apps/prod/tekton/configs/triggers/triggers/kustomization.yaml

@@ -1,13 +1,13 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - pingcap-qe/artifacts/create-tag-pingcap-qe-artifacts.yaml
-  - pingcap-qe/artifacts/pull-pingcap-qe-artifacts.yaml
-  - pingcap-qe/artifacts/push-pingcap-qe-artifacts.yaml
+  # - pingcap-qe/artifacts/create-tag-pingcap-qe-artifacts.yaml
+  # - pingcap-qe/artifacts/pull-pingcap-qe-artifacts.yaml
+  # - pingcap-qe/artifacts/push-pingcap-qe-artifacts.yaml
   - pingcap/create-branch-product-components-hotfix.yaml
   # - pingcap/tidb/push.yaml # passed
-  - pingcap/tiflash/push.yaml
+  # - pingcap/tiflash/push.yaml
   # - pingcap/tiflow/push.yaml # passed
-  - tikv/pd/create-tag.yaml
-  # - tikv/pd/push.yaml # passed
+  # - tikv/pd/create-tag.yaml
+  - tikv/pd/push.yaml # passed
   - tikv/tikv/push.yaml