[Feature][Test]Add E2E Test Programing process (#4036)

Co-authored-by: zackyoungh <zackyoungh@users.noreply.github.com>
DataLinkDC · Dec 13, 2024 · 59c142e · 59c142e
1 parent 0d980ad
commit 59c142e
Show file tree

Hide file tree

Showing 25 changed files with 1,145 additions and 3 deletions.
diff --git a/.github/workflows/e2e_test.yml b/.github/workflows/e2e_test.yml
@@ -0,0 +1,83 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: E2E Test
+
+on:
+  workflow_dispatch:
+
+jobs:
+  init_env:
+    name: init env
+    runs-on: ubuntu-latest
+    services:
+      registry:
+        image: registry:2
+        ports:
+          - 5000:5000
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Init  Env Jar
+        run: |
+          wget -O e2e_test/docker-compose-env/dinky/mysql-connector-java-8.0.30.jar  https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.30/mysql-connector-java-8.0.30.jar &&
+          wget -O e2e_test/docker-compose-env/flink/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar  https://repo1.maven.org/maven2/org/apache/flink/flink-shaded-hadoop-2-uber/2.8.3-10.0/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar
+      - name: Init  Docker Network
+        run: |
+          docker network create -d bridge dinky_net
+      - name: Init Run Docker MySQL
+        uses: hoverkraft-tech/compose-action@v2.0.2
+        with:
+          compose-file: ./e2e_test/docker-compose-env/mysql/docker-compose.yml
+      - name: Init Run Docker Hadoop
+        uses: hoverkraft-tech/compose-action@v2.0.2
+        with:
+          compose-file: ./e2e_test/docker-compose-env/hadoop/docker-compose.yml
+      # 设置 QEMU, 后面 docker buildx 依赖此.
+      - name: Init Run Docker Flink
+        uses: hoverkraft-tech/compose-action@v2.0.2
+        with:
+          compose-file: ./e2e_test/docker-compose-env/flink/docker-compose.yml
+      # 设置 QEMU, 后面 docker buildx 依赖此.
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      # 设置 Docker buildx, 方便构建 Multi platform 镜像
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver-opts: network=host
+      - name: Build Dinky Image
+        uses: docker/build-push-action@v5
+        with:
+          file: ./deploy/docker/Dockerfile
+          # 是否 docker push
+          push: true
+          build-args: |
+            FLINK_VERSION=1.14
+            DINKY_VERSION=1
+          tags: |
+            localhost:5000/dinky/dinky-test:flink-1.14
+      - name: Init Run Docker Dinky
+        uses: hoverkraft-tech/compose-action@v2.0.2
+        with:
+          compose-file: ./e2e_test/docker-compose-env/dinky/docker-compose.yml
+      - name:  Run Docker Python Script
+#        uses: hoverkraft-tech/compose-action@v2.0.2
+#        with:
+#          compose-file: ./e2e_test/tools/docker-compose.yml
+        run: |
+          sleep 30 && cd ./e2e_test/tools && docker run  -v ./:/app -w /app   --net dinky_net --rm --entrypoint /bin/bash python:3.9 -c  'pip install  -r requirements.txt  &&  python main.py dinky14:8888'
diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile
@@ -27,7 +27,7 @@ COPY --from=ui-build /build/dist/ /build/dinky-web/dist/
 
 RUN mvn package -Dmaven.test.skip=true -P prod,flink-single-version,flink-${FLINK_VERSION},fast
 RUN mkdir release &&  \
-    tar -C release -xvf build/dinky-release-${FLINK_VERSION}-${DINKY_VERSION}.tar.gz && \
+    tar -C release -xvf build/dinky-release-${FLINK_VERSION}-*.tar.gz && \
     mv release/dinky-release-* release/dinky
 
 
@@ -48,8 +48,8 @@ RUN rm -f  /opt/dinky/extends/flink${FLINK_VERSION}/flink/flink-table-planner-lo
 
 COPY --from=flink-base  /opt/flink/opt/flink-table-planner*.jar /opt/dinky/extends/flink${FLINK_VERSION}/flink/
 
-RUN mkdir /opt/dinky/customJar && chmod -R 777 /opt/dinky/ && sed -i 's/-Xms512M -Xmx2048M -XX:PermSize=512M/-XX:+UseContainerSupport -XX:InitialRAMPercentage=70.0 -XX:MaxRAMPercentage=70.0/g' auto.sh
+RUN mkdir /opt/dinky/customJar && chmod -R 777 /opt/dinky/ && sed -i 's/-Xms512M -Xmx2048M -XX:PermSize=512M/-XX:+UseContainerSupport -XX:InitialRAMPercentage=70.0 -XX:MaxRAMPercentage=70.0/g' ./bin/auto.sh
 
 EXPOSE 8888
 
-CMD  ./bin/auto.sh startOnPending
+CMD  ./bin/auto.sh startOnPending
diff --git a/dinky-admin/src/main/java/org/dinky/Dinky.java b/dinky-admin/src/main/java/org/dinky/Dinky.java
@@ -19,6 +19,7 @@
 
 package org.dinky;
 
+import org.dinky.data.constant.DirConstant;
 import org.dinky.security.NoExitSecurityManager;
 
 import org.springframework.boot.SpringApplication;
@@ -29,6 +30,7 @@
 
 import com.alibaba.druid.proxy.DruidDriver;
 
+import cn.hutool.core.io.FileUtil;
 import lombok.SneakyThrows;
 
 /**
@@ -54,6 +56,8 @@ public static void main(String[] args) {
         // chinese: 初始化JDBC Driver，因为包的数量特别庞大，所以这里需要异步执行，并提前加载Driver
         new Thread(DruidDriver::getInstance).start();
 
+        FileUtil.mkdir(DirConstant.getTempRootDir());
+
         SpringApplication app = new SpringApplication(Dinky.class);
         app.run(args);
     }

diff --git a/e2e_test/docker-compose-env/dinky/docker-compose.yml b/e2e_test/docker-compose-env/dinky/docker-compose.yml
@@ -0,0 +1,19 @@
+version: "3"
+networks:
+  dinky_net:
+    external: true
+services:
+  dinky14:
+    restart: always
+    image: localhost:5000/dinky/dinky-test:flink-1.14
+    environment:
+      - DB_ACTIVE=mysql
+      - MYSQL_ADDR=mysql:3306
+      - MYSQL_DATABASE=dinky
+      - MYSQL_USERNAME=root
+      - MYSQL_PASSWORD=dinky
+    volumes:
+      - ./mysql-connector-java-8.0.30.jar:/opt/dinky/lib/mysql-connector-java-8.0.30.jar
+      - ../flink/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar:/opt/dinky/lib/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar
+    networks:
+      - dinky_net
diff --git a/e2e_test/docker-compose-env/flink/conf/core-site.xml b/e2e_test/docker-compose-env/flink/conf/core-site.xml
@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+    <property><name>hadoop.proxyuser.hue.hosts</name><value>*</value></property>
+    <property><name>fs.defaultFS</name><value>hdfs://namenode:9000</value></property>
+    <property><name>hadoop.http.staticuser.user</name><value>root</value></property>
+    <property><name>io.compression.codecs</name><value>org.apache.hadoop.io.compress.SnappyCodec</value></property>
+    <property><name>hadoop.proxyuser.hue.groups</name><value>*</value></property>
+</configuration>