From 2d2e3c7baa3448a1051d0eef4923a2213f5e9008 Mon Sep 17 00:00:00 2001 From: catpineapple <1391869588@qq.com> Date: Mon, 12 Aug 2024 19:18:04 +0800 Subject: [PATCH] how to build doris image --- docker/runtime/README.md | 137 ++++++++++++++++++++++++++ docker/runtime/k8s/doris_be.yml | 105 -------------------- docker/runtime/k8s/doris_cn.yml | 106 -------------------- docker/runtime/k8s/doris_follower.yml | 131 ------------------------ 4 files changed, 137 insertions(+), 342 deletions(-) create mode 100644 docker/runtime/README.md delete mode 100644 docker/runtime/k8s/doris_be.yml delete mode 100644 docker/runtime/k8s/doris_cn.yml delete mode 100644 docker/runtime/k8s/doris_follower.yml diff --git a/docker/runtime/README.md b/docker/runtime/README.md new file mode 100644 index 00000000000000..189668bb4813a4 --- /dev/null +++ b/docker/runtime/README.md @@ -0,0 +1,137 @@ + + +## How to build a docker image + +### Preparation + +#### Download and copy the Doris code repo + +```console +$ cd /to/your/workspace/ +$ git clone https://github.com/apache/doris.git +$ cd doris/docker +$ cp -r runtime/ /to/your/workspace/ +``` + +After preparation, your workspace should like this: + +``` +. +├── runtime +│   ├── README.md +│   ├── all-in-one +│   ├── base-image +│   ├── be +│   ├── broker +│   ├── docker-compose-demo +│   ├── doris-compose +│   ├── fe +│   ├── ms +``` + +#### Download, unzip and copy the antidote package + +Here we take the build of x64 (avx2) and arm64 platforms of 2.1.5 version as an example. + +1. Download binary + +Go to the doris [official website](https://doris.apache.org/download) to download the binary package you need, and pay attention to selecting the doris version and architecture type you need. + +```console +$ wget https://apache-doris-releases.oss-accelerate.aliyuncs.com/apache-doris-2.1.5-bin-x64.tar.gz +# or +$ wget https://apache-doris-releases.oss-accelerate.aliyuncs.com/apache-doris-2.1.5-bin-arm64.tar.gz +``` + +2. Unzip binary package + +```console +$ tar -zxvf apache-doris-2.1.5-bin-x64.tar.gz +# or +$ tar -zxvf apache-doris-2.1.5-bin-arm64.tar.gz +``` + +3. copy the antidote package + +You need to copy the corresponding directories to different files in sequence as shown in the table: + +amd64(avx2) platform: + +| doris type | doris package | docker file path | +|:----------:|:--------------------------------------------------------:|:--------------------------------------------------------------------------------------:| +| fe | apache-doris-2.1.5-bin-x64/fe | runtime/fe/resource/amd64/apache-doris-2.1.5-bin-x64/fe | +| be | apache-doris-2.1.5-bin-x64/be | runtime/be/resource/amd64/apache-doris-2.1.5-bin-x64/be | +| ms | apache-doris-2.1.5-bin-x64/ms | runtime/ms/resource/amd64/apache-doris-2.1.5-bin-x64/ms | +| broker | apache-doris-2.1.5-bin-x64/extensions/apache_hdfs_broker | runtime/broker/resource/amd64/apache-doris-2.1.5-bin-x64/extensions/apache_hdfs_broker | + +arm64 platform: + +| doris type | doris package | docker file path | +|:----------:|:----------------------------------------------------------:|:----------------------------------------------------------------------------------------:| +| fe | apache-doris-2.1.5-bin-arm64/fe | runtime/fe/resource/arm64/apache-doris-2.1.5-bin-arm64/fe | +| be | apache-doris-2.1.5-bin-arm64/be | runtime/be/resource/arm64/apache-doris-2.1.5-bin-arm64/be | +| ms | apache-doris-2.1.5-bin-arm64/ms | runtime/ms/resource/arm64/apache-doris-2.1.5-bin-arm64/ms | +| broker | apache-doris-2.1.5-bin-arm64/extensions/apache_hdfs_broker | runtime/broker/resource/arm64/apache-doris-2.1.5-bin-arm64/extensions/apache_hdfs_broker | + + + +### Base image + +**NOTICE** + +The below images depend on the base image selectdb/base. If your environment cannot access it, you can pre-build the base image. The base image contains the basic environment for doris to run, including JDK, openssl, etc. + +#### Build base image + +As mentioned in the preparation steps above, the Dockerfile of the Base image is under the runtime/base-image path. To build it, make sure you have pulled the Doris code repo and execute the following command. + +```console +$ cd /to/your/workspace/runtime/base-image && docker build . -t doris-base:latest -f Dockerfile_base +``` + +#### Adjust Dockerfile +Adjust the base image name used by the Dockerfile of the doris component and replace the base image you built yourself, As shown in the following example: + +```dockerfile +... +# Adjust the base image here +FROM selectdb/base:latest + +ARG TARGETARCH + +ARG DORIS_VERSION="x.x.x" +... +``` + + +### Build Doris docker image + +Execute the following command, Docker will automatically confirm the architecture type + +```console +$ cd /to/your/workspace/runtime/fe && docker build . -t doris.fe:2.1.5 -f Dockerfile --build-arg DORIS_VERSION=2.1.5 +$ cd /to/your/workspace/runtime/be && docker build . -t doris.be:2.1.5 -f Dockerfile --build-arg DORIS_VERSION=2.1.5 +$ cd /to/your/workspace/runtime/ms && docker build . -t doris.ms:2.1.5 -f Dockerfile --build-arg DORIS_VERSION=2.1.5 +$ cd /to/your/workspace/runtime/broker && docker build . -t doris.broker:2.1.5 -f Dockerfile --build-arg DORIS_VERSION=2.1.5 +``` + +### Latest update time + +2024-8-12 diff --git a/docker/runtime/k8s/doris_be.yml b/docker/runtime/k8s/doris_be.yml deleted file mode 100644 index 84c7f2a0da85d1..00000000000000 --- a/docker/runtime/k8s/doris_be.yml +++ /dev/null @@ -1,105 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -apiVersion: v1 -kind: Service -metadata: - name: doris-be-cluster1 - labels: - app: doris-be-cluster1 -spec: - ports: - - port: 9060 - name: be-port - - port: 8040 - name: webserver-port - - port: 9050 - name: heartbeat-port #This name should be fixed. Doris will get the port information through this name - - port: 8060 - name: brpc-port - clusterIP: None - selector: - app: doris-be-cluster1 ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: doris-be-cluster1 - labels: - app: doris-be-cluster1 -spec: - selector: - matchLabels: - app: doris-be-cluster1 - serviceName: doris-be-cluster1 - replicas: 3 - template: - metadata: - name: doris-be-cluster1 - labels: - app: doris-be-cluster1 - spec: - containers: - - name: doris-be-cluster1 - #Need to change to real mirror information - image: apache-doris-be:test - imagePullPolicy: IfNotPresent - env: - #Specify the startup type as k8s to bypass some restrictions of the official image initialization script - - name: BUILD_TYPE - value: "k8s" - ports: - - containerPort: 9060 - name: be-port - - containerPort: 8040 - name: webserver-port - - containerPort: 9050 - name: heartbeat-port - - containerPort: 8060 - name: brpc-port - volumeMounts: - #Mount the configuration file in the way of configmap - - name: conf - mountPath: /opt/apache-doris/be/conf - #Ifnot mounted, when enable_profile, error will be reported when querying the data from jdbc catalog - #Error message: error setting certificate verify locations: CAfile:/etc/pki/tls/certs/ca-bundle.crt CApath: none - - name: sys - mountPath: /etc/pki - readOnly: true - volumes: - - name: conf - configMap: - name: be-conf - - name: sys - hostPath: - path: /etc/pki ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: be-conf -data: - be.conf: | - PPROF_TMPDIR="$DORIS_HOME/log/" - sys_log_level = INFO - - be_port = 9060 - webserver_port = 8040 - heartbeat_service_port = 9050 - brpc_port = 8060 - - priority_networks = 172.16.0.0/24 \ No newline at end of file diff --git a/docker/runtime/k8s/doris_cn.yml b/docker/runtime/k8s/doris_cn.yml deleted file mode 100644 index 391df11162b999..00000000000000 --- a/docker/runtime/k8s/doris_cn.yml +++ /dev/null @@ -1,106 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -apiVersion: v1 -kind: Service -metadata: - name: doris-cn-cluster1 - labels: - app: doris-cn-cluster1 -spec: - ports: - - port: 9060 - name: be-port - - port: 8040 - name: webserver-port - - port: 9050 - name: heartbeat-port #This name should be fixed. Doris will get the port information through this name - - port: 8060 - name: brpc-port - clusterIP: None - selector: - app: doris-cn-cluster1 ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: doris-cn-cluster1 - labels: - app: doris-cn-cluster1 -spec: - selector: - matchLabels: - app: doris-cn-cluster1 - serviceName: doris-cn-cluster1 - replicas: 3 - template: - metadata: - name: doris-cn-cluster1 - labels: - app: doris-cn-cluster1 - spec: - containers: - - name: doris-cn-cluster1 - #Need to change to real mirror information - image: apache-doris-be:test - imagePullPolicy: IfNotPresent - env: - #Specify the startup type as k8s to bypass some restrictions of the official image initialization script - - name: BUILD_TYPE - value: "k8s" - ports: - - containerPort: 9060 - name: be-port - - containerPort: 8040 - name: webserver-port - - containerPort: 9050 - name: heartbeat-port - - containerPort: 8060 - name: brpc-port - volumeMounts: - #Mount the configuration file in the way of configmap - - name: conf - mountPath: /opt/apache-doris/be/conf - #Ifnot mounted, when enable_profile, error will be reported when querying the data from jdbc catalog - #Error message: error setting certificate verify locations: CAfile:/etc/pki/tls/certs/ca-bundle.crt CApath: none - - name: sys - mountPath: /etc/pki - readOnly: true - volumes: - - name: conf - configMap: - name: cn-conf - - name: sys - hostPath: - path: /etc/pki ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: cn-conf -data: - be.conf: | - PPROF_TMPDIR="$DORIS_HOME/log/" - sys_log_level = INFO - - be_port = 9060 - webserver_port = 8040 - heartbeat_service_port = 9050 - brpc_port = 8060 - #Specify node type as calculation node - be_node_role = computation - priority_networks = 172.16.0.0/24 \ No newline at end of file diff --git a/docker/runtime/k8s/doris_follower.yml b/docker/runtime/k8s/doris_follower.yml deleted file mode 100644 index b22915fe249a74..00000000000000 --- a/docker/runtime/k8s/doris_follower.yml +++ /dev/null @@ -1,131 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -apiVersion: v1 -kind: Service -metadata: - name: doris-follower-cluster1 - labels: - app: doris-follower-cluster1 -spec: - ports: - - port: 8030 - name: http-port - - port: 9020 - name: rpc-port - - port: 9030 - name: query-port - - port: 9010 - name: edit-log-port #This name should be fixed. Doris will get the port information through this name - clusterIP: None - selector: - app: doris-follower-cluster1 ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: doris-follower-cluster1 - labels: - app: doris-follower-cluster1 -spec: - selector: - matchLabels: - app: doris-follower-cluster1 - serviceName: doris-follower-cluster1 - replicas: 3 - template: - metadata: - name: doris-follower-cluster1 - labels: - app: doris-follower-cluster1 - spec: - containers: - - name: doris-follower-cluster1 - #Need to change to real mirror information - image: apache-doris-fe:test - imagePullPolicy: IfNotPresent - env: - #Specify the startup type as k8s to bypass some restrictions of the official image initialization script - - name: BUILD_TYPE - value: "k8s" - #Initialize the fe of three nodes - - name: FE_INIT_NUMBER - value: "3" - #ServiceName of bakend_cn node,（if do not have bakend_cn node,do not configure this environment variable） - - name: CN_SERVICE - value: "doris-cn-cluster1" - #StatefulSetName of bakend_cn node,（if do not have bakend_cn node,do not configure this environment variable) - - name: CN_STATEFULSET - value: "doris-cn-cluster1" - #ServiceName of bakend node,（if do not have bakend node,do not configure this environment variable） - - name: BE_SERVICE - value: "doris-be-cluster1" - #StatefulSetName of bakend node,（if do not have bakend node,do not configure this environment variable) - - name: BE_STATEFULSET - value: "doris-be-cluster1" - #ServiceName of follower node,（if do not have follower node,do not configure this environment variable） - - name: FE_SERVICE - value: "doris-follower-cluster1" - ##StatefulSetName of follower node,（if do not have follower node,do not configure this environment variable) - - name: FE_STATEFULSET - value: "doris-follower-cluster1" - ports: - - containerPort: 8030 - name: http-port - - containerPort: 9020 - name: rpc-port - - containerPort: 9030 - name: query-port - - containerPort: 9010 - name: edit-log-port - volumeMounts: - #Mount the configuration file in the way of configmap - - name: conf - mountPath: /opt/apache-doris/fe/conf - #In order to call the api of k8s - - name: kube - mountPath: /root/.kube/config - readOnly: true - volumes: - - name: conf - configMap: - name: follower-conf - - name: kube - hostPath: - path: /root/.kube/config ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: follower-conf -data: - fe.conf: | - priority_networks = 172.16.0.0/24 - #It can automatically maintain node information by getting the number of replicas of StatefulSet, similar to alter system add/drop back - enable_deploy_manager = k8s - #Automatically adjust the IP of the node according to the domain name (for example, after the pod is restarted, the domain name is still doris-be-cluster1-0-doris-be-cluster1.default.svc.cluster.local, but the IP may change from 172.16.0.9 to 172.16.0.10) - enable_fqdn_mode = true - LOG_DIR = ${DORIS_HOME}/log - sys_log_level = INFO - http_port = 8030 - rpc_port = 9020 - query_port = 9030 - edit_log_port = 9010 - #Doris needs to generate the log4j configuration file according to the fe.yml configuration information, which is written in the same directory as fe.yml by default, but the config we mount is readonly, so specify this configuration to write the log4j file to another location - custom_config_dir = /opt/apache-doris/ - #when set to false, the backend will not be dropped and remaining in DECOMMISSION state - drop_backend_after_decommission = false