[Feature][E2E] E2E Test the new FLINK YARN environment mode (#4064)

Co-authored-by: zackyoungh <zackyoungh@users.noreply.github.com>
DataLinkDC · Dec 21, 2024 · 58bf7f1 · 58bf7f1
1 parent e4240c3
commit 58bf7f1
Show file tree

Hide file tree

Showing 17 changed files with 647 additions and 267 deletions.
diff --git a/.github/workflows/backend.yaml b/.github/workflows/backend.yaml
@@ -19,6 +19,7 @@ name: Backend
 
 on:
   push:
+  workflow_dispatch:
   pull_request:
     branches:
       - dev
@@ -57,17 +58,17 @@ jobs:
             frontend:
               - 'dinky-web/**'
       - uses: actions/setup-node@v3
-        if:  steps.filter.outputs.frontend == 'true'
+        if: steps.filter.outputs.frontend == 'true'
         with:
           node-version: 16
       - name: Get npm cache directory
         id: npm-cache-dir
-        if:  steps.filter.outputs.frontend == 'true'
+        if: steps.filter.outputs.frontend == 'true'
         run: |
           echo "::set-output name=dir::$(npm config get cache)"
       - uses: actions/cache@v3
         id: npm-cache # use this to check for `cache-hit` ==> if: steps.npm-cache.outputs.cache-hit != 'true'
-        if:  steps.filter.outputs.frontend == 'true'
+        if: steps.filter.outputs.frontend == 'true'
         with:
           path: |
             ${{ steps.npm-cache-dir.outputs.dir }}
@@ -76,20 +77,25 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-node-
       - name: Install Dependencies
-        if:  steps.filter.outputs.frontend == 'true'
+        if: steps.filter.outputs.frontend == 'true'
         run: cd dinky-web && npm install --no-audit --progress=false --legacy-peer-deps
       - name: Npm Web Build
-        if:  steps.filter.outputs.frontend == 'true'
+        if: steps.filter.outputs.frontend == 'true'
         run: cd dinky-web && npm run build
-  build_jdk:
-    name: Build_JDK
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: dinky-web
+          path: ./dinky-web/dist
+  build_release:
+    name: Build Release
     runs-on: ubuntu-latest
     needs: check
     strategy:
       fail-fast: true
       matrix:
-        jdk: [8, 11]
-        flink: ['1.14', '1.15', '1.16', '1.17', '1.18', '1.19', '1.20']
+        jdk: [ 8, 11 ]
+        flink: [ '1.14', '1.15', '1.16', '1.17', '1.18', '1.19', '1.20' ]
 
     timeout-minutes: 30
     env:
@@ -100,7 +106,7 @@ jobs:
       - name: Set up JDK ${{ matrix.jdk }}
         uses: actions/setup-java@v2
         with:
-          java-version:  ${{ matrix.jdk }}
+          java-version: ${{ matrix.jdk }}
           distribution: 'adopt'
       - name: Cache local Maven repository
         uses: actions/cache@v3
@@ -132,6 +138,87 @@ jobs:
       - name: Check package size
         run: |
           ./check_package_size.sh
-#      - name: Run Unit tests
-#        run: |
-#          ./mvnw -T 2C -B clean verify -Dmaven.test.skip=false -Dgpg.skip=true --no-snapshot-updates
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        if: ${{ matrix.jdk  == 8 }}
+        with:
+          name: dinky-realease-${{ matrix.flink }}
+          path: ./build/dinky-release*.tar.gz
+  #      - name: Run Unit tests
+  #        run: |
+  #          ./mvnw -T 2C -B clean verify -Dmaven.test.skip=false -Dgpg.skip=true --no-snapshot-updates
+
+
+  run_e2e:
+    name: Run E2E
+    needs: build_release
+    strategy:
+      fail-fast: true
+      matrix:
+        flink: [ '1.14', '1.15', '1.16', '1.17', '1.18', '1.19', '1.20' ]
+    runs-on: ubuntu-latest
+    services:
+      registry:
+        image: registry:2
+        ports:
+          - 5000:5000
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Download artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: dinky-realease-${{ matrix.flink }}
+          path: ./build
+      # 设置 QEMU, 后面 docker buildx 依赖此.
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      # 设置 Docker buildx, 方便构建 Multi platform 镜像
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver-opts: network=host
+      - name: Build Dinky Image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ./e2e_test/docker-compose-env/Dockerfile
+          # 是否 docker push
+          push: true
+          build-args: |
+            FLINK_VERSION=${{ matrix.flink }}
+          tags: |
+            localhost:5000/dinky/dinky-test:flink
+      - name: Init  Env Jar
+        run: |
+          wget -O e2e_test/docker-compose-env/dinky/mysql-connector-java-8.0.30.jar  https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.30/mysql-connector-java-8.0.30.jar &&
+          wget -O e2e_test/docker-compose-env/flink/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar  https://repo1.maven.org/maven2/org/apache/flink/flink-shaded-hadoop-2-uber/2.8.3-10.0/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar &&
+          wget -O e2e_test/docker-compose-env/dinky/javax.ws.rs-api-2.1.1.jar  https://repo1.maven.org/maven2/javax/ws/rs/javax.ws.rs-api/2.1.1/javax.ws.rs-api-2.1.1.jar
+      - name: Init  Docker Network
+        run: |
+          docker network create -d bridge dinky_net
+      - name: Init Run Docker MySQL
+        uses: hoverkraft-tech/compose-action@v2.0.2
+        with:
+          compose-file: ./e2e_test/docker-compose-env/mysql/docker-compose.yml
+      - name: Init Run Docker Dinky
+        uses: hoverkraft-tech/compose-action@v2.0.2
+        with:
+          compose-file: ./e2e_test/docker-compose-env/dinky/docker-compose.yml
+      - name: Init Run Docker Hadoop
+        uses: hoverkraft-tech/compose-action@v2.0.2
+        with:
+          compose-file: ./e2e_test/docker-compose-env/hadoop/docker-compose.yml
+      - name: Replace Flink  docker-compose yml
+        run: |
+          export FLINK_VERSION=${{ matrix.flink }} && envsubst < ./e2e_test/docker-compose-env/flink/docker-compose.yml > ./e2e_test/docker-compose-env/flink/docker-compose-${{ matrix.flink }}.yml
+      - name: Init Run Docker Flink
+        uses: hoverkraft-tech/compose-action@v2.0.2
+        with:
+          compose-file: ./e2e_test/docker-compose-env/flink/docker-compose-${{ matrix.flink }}.yml
+
+      - name: Cp Flink Jar Deps
+        run: docker cp dinky:/opt/dinky/ ./dinky-release
+      - name: Run Docker Python Script
+        run: |
+          docker run -v ./dinky-release/extends/flink${{ matrix.flink }}:/flink/lib -v ./e2e_test/docker-compose-env/dinky/mysql-connector-java-8.0.30.jar:/flink/lib/mysql-connector-java-8.0.30.jar -v./e2e_test/docker-compose-env/flink/conf:/flink/conf -v ./dinky-release/jar:/dinky/jar  -v./e2e_test/tools:/app -w /app   --net dinky_net --rm --entrypoint /bin/bash python:3.9 -c  'pip install  -r requirements.txt  &&  python main.py dinky:8888'
diff --git a/.github/workflows/docker_build.yaml b/.github/workflows/docker_build.yaml
@@ -34,7 +34,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        flink: [1.14,1.15,1.16,1.17,1.18,1.19]
+        flink: ["1.14","1.15","1.16","1.17","1.18","1.19","1.20"]
     steps:
       # git checkout 代码
       - name: Checkout

diff --git a/.github/workflows/e2e_test.yml b/.github/workflows/e2e_test.yml
diff --git a/e2e_test/docker-compose-env/Dockerfile b/e2e_test/docker-compose-env/Dockerfile
@@ -0,0 +1,28 @@
+ARG FLINK_VERSION
+
+FROM flink:${FLINK_VERSION}-scala_2.12-java8 as flink-base
+
+
+FROM eclipse-temurin:8-jre-jammy
+
+ARG FLINK_VERSION
+ENV FLINK_VERSION=${FLINK_VERSION}
+ENV DINKY_HOME=/opt/dinky/
+ENV H2_DB=./tmp/db/h2
+ADD build/dinky-release*.tar.gz /opt
+RUN ls /opt && mv /opt/dinky-release* /opt/dinky
+
+
+WORKDIR /opt/dinky/
+
+USER root
+COPY --from=flink-base  /opt/flink/lib/*.jar  /opt/dinky/extends/flink${FLINK_VERSION}/flink/
+RUN rm -f  /opt/dinky/extends/flink${FLINK_VERSION}/flink/flink-table-planner-loader*.jar
+
+COPY --from=flink-base  /opt/flink/opt/flink-table-planner*.jar /opt/dinky/extends/flink${FLINK_VERSION}/flink/
+
+RUN mkdir /opt/dinky/customJar && chmod -R 777 /opt/dinky/ && sed -i 's/-Xms512M -Xmx2048M -XX:PermSize=512M/-XX:+UseContainerSupport -XX:InitialRAMPercentage=70.0 -XX:MaxRAMPercentage=70.0/g' ./bin/auto.sh
+
+EXPOSE 8888
+
+CMD  ./bin/auto.sh startOnPending
diff --git a/e2e_test/docker-compose-env/dinky/docker-compose.yml b/e2e_test/docker-compose-env/dinky/docker-compose.yml
@@ -3,9 +3,10 @@ networks:
   dinky_net:
     external: true
 services:
-  dinky14:
+  dinky:
+    container_name: dinky
     restart: always
-    image: localhost:5000/dinky/dinky-test:flink-1.14
+    image: localhost:5000/dinky/dinky-test:flink
     environment:
       - DB_ACTIVE=mysql
       - MYSQL_ADDR=mysql:3306
@@ -14,6 +15,9 @@ services:
       - MYSQL_PASSWORD=dinky
     volumes:
       - ./mysql-connector-java-8.0.30.jar:/opt/dinky/lib/mysql-connector-java-8.0.30.jar
+      - ./javax.ws.rs-api-2.1.1.jar:/opt/dinky/lib/javax.ws.rs-api-2.1.1.jar
       - ../flink/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar:/opt/dinky/lib/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar
+      - ../flink/conf/log4j-console.properties:/flink/conf/log4j-console.properties
+      - ../hadoop:/flink/conf
     networks:
       - dinky_net
diff --git a/e2e_test/docker-compose-env/flink/conf/flink-conf.yaml b/e2e_test/docker-compose-env/flink/conf/flink-conf.yaml
@@ -272,8 +272,5 @@ classloader.resolve-order: parent-first
 # Interval in milliseconds for refreshing the monitored directories.
 #historyserver.archive.fs.refresh-interval: 10000
 
-blob.server.port: 6124
-query.server.port: 6125
-
 jobmanager.rpc.address: jobmanager
 env.java.opts: "-Dfile.encoding=UTF-8"
diff --git a/e2e_test/docker-compose-env/flink/docker-compose.yml b/e2e_test/docker-compose-env/flink/docker-compose.yml
@@ -4,8 +4,10 @@ networks:
     external: true
 services:
   jobmanager:
+    hostname: jobmanager
+    container_name: jobmanager
     restart: always
-    image: flink:1.14.6
+    image: flink:${FLINK_VERSION}-scala_2.12-java8
     command: jobmanager
     environment:
       - HADOOP_CONF_DIR=/opt/flink/conf
@@ -15,7 +17,9 @@ services:
     networks:
       - dinky_net
   taskmanager:
-    image: flink:1.14.6
+    hostname: taskmanager
+    container_name: taskmanager
+    image: flink:${FLINK_VERSION}-scala_2.12-java8
     command: taskmanager
     environment:
       - HADOOP_CONF_DIR=/opt/flink/conf
@@ -24,3 +28,5 @@ services:
       - ./flink-shaded-hadoop-2-uber-2.8.3-10.0.jar:/opt/flink/lib/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar
     networks:
       - dinky_net
+    depends_on:
+      - jobmanager