update paddlex ci (#2990)

PaddlePaddle · Nov 27, 2024 · 7815d61 · 7815d61
1 parent 7738bd0
commit 7815d61
Show file tree

Hide file tree

Showing 4 changed files with 89 additions and 62 deletions.
diff --git a/models/PaddleX/ci/black_list.txt b/models/PaddleX/ci/black_list.txt
@@ -1,36 +1,11 @@
 ################################### All ##################################
-All:PP-YOLOE_plus_SOD-largesize-L
 ################################## Train #################################
 ################################ Evaluate ################################
 ################################# Predict ################################
-Predict:PP-YOLOE_seg-S
-Predict:MaskRCNN-ResNet50
-Predict:MaskRCNN-ResNet50-FPN
-Predict:MaskRCNN-ResNet50-vd-FPN
-Predict:MaskRCNN-ResNet101-FPN
-Predict:MaskRCNN-ResNet101-vd-FPN
-Predict:MaskRCNN-ResNeXt101-vd-FPN
-Predict:Cascade-MaskRCNN-ResNet50-vd-SSLDv2-FPN
-Predict:Cascade-MaskRCNN-ResNet50-FPN
-Predict:Nonstationary_ad
 Predict:TimesNet_ad
-Predict:PatchTST_ad
-Predict:DLinear_ad
-Predict:PP-YOLOE_plus_SOD-L
-Predict:PP-YOLOE_plus_SOD-S
-Predict:Cascade-FasterRCNN-ResNet50-vd-SSLDv2-FPN
-Predict:Cascade-FasterRCNN-ResNet50-FPN
-Predict:FasterRCNN-Swin-Tiny-FPN
-Predict:FasterRCNN-ResNeXt101-vd-FPN
-Predict:FasterRCNN-ResNet101-FPN
-Predict:FasterRCNN-ResNet101
-Predict:FasterRCNN-ResNet50-vd-SSLDv2-FPN
-Predict:FasterRCNN-ResNet50-vd-FPN
-Predict:FasterRCNN-ResNet50-FPN
-Predict:FasterRCNN-ResNet50
-Predict:FasterRCNN-ResNet34-FPN
 ################################## Export ################################
 ################################# Pipeline ###############################
 Pipeline:PP-ChatOCRv3-doc
 Pipeline:PP-ShiTuV2
 Pipeline:face_recognition
+Pipeline:ts_ad
diff --git a/models/PaddleX/ci/checker.py b/models/PaddleX/ci/checker.py
@@ -227,8 +227,8 @@ def check_dataset_json_content(self, output_dir, module_name, dataset_result_jso
                     pass_flag = False
         else:
             show_type = json_data.get("show_type")
-            if show_type not in ["image", "txt", "csv"]:
-                check_dataset_json_message.append(f"检查失败：{show_type} 必须为'image', 'txt', 'csv'其中一个")
+            if show_type not in ["image", "txt", "csv", "video"]:
+                check_dataset_json_message.append(f"检查失败：{show_type} 必须为'image', 'txt', 'csv','video'其中一个")
                 pass_flag = False
 
             if module_name == "general_recognition":
@@ -249,6 +249,7 @@ def check_dataset_json_content(self, output_dir, module_name, dataset_result_jso
                 or "table" in module_name
                 or "formula" in module_name
                 or module_name == "general_recognition"
+                or module_name == "face_feature"
             ):
                 try:
                     num_class = int(json_data["attributes"].get("num_classes"))
@@ -368,7 +369,7 @@ def run_checks(self, args):
 
         if args.check_train_result_json:
             # 检查 train_result.json 内容
-            train_result_json = os.path.join(output_dir, "train_results.json")
+            train_result_json = os.path.join(output_dir, "train_result.json")
             check_weights_items = args.check_weights_items
             check_train_json_message = []
             check_train_json_flag, check_train_json_message = self.check_train_json_content(

diff --git a/models/PaddleX/ci/ci_run.sh b/models/PaddleX/ci/ci_run.sh
@@ -11,8 +11,8 @@ MODEL_LIST_FILE=$2
 # MD_NUM: PR中MD文件改动数量，用于判断是否需要进行文档超链接检测，默认为空，设置示例：export MD_NUM=10
 # WITHOUT_MD_NUM: PR中MD文件改动之外的改动文件数量，用于判断进行文档超链接检测后是否进行正常的CI，默认为空，设置示例：export WITHOUT_MD_NUM=10
 
-# set -x
 if [[ $MODE == 'PaddleX' ]];then
+    set -x
     failed_cmd_list=""
 fi
 
@@ -52,25 +52,25 @@ function run_command(){
     printf "\e[32m|%-20s| %-50s | %-20s\n\e[0m" "[${time_stamp}]" "${command}"
     eval $command
     last_status=${PIPESTATUS[0]}
+    n=1
+    # Try 2 times to run command if it fails
     if [[ $MODE != 'PaddleX' ]];then
-        n=1
-        # Try 3 times to run command if it fails
         while [[ $last_status != 0 ]]; do
             sleep 10
             n=`expr $n + 1`
             printf "\e[32m|%-20s| %-50s | %-20s\n\e[0m" "[${time_stamp}]" "${command}"
+            sync
+            echo 1 > /proc/sys/vm/drop_caches
             eval $command
             last_status=${PIPESTATUS[0]}
-            if [[ $n -eq 3 && $last_status != 0 ]]; then
-                echo "Retry 3 times failed with command: ${command}"
+            if [[ $n -eq 2 && $last_status != 0 ]]; then
+                echo "Retry 2 times failed with command: ${command}"
                 exit 1
             fi
         done
-    else
-        if [[ $last_status != 0 ]];then
-            failed_cmd_list="$failed_cmd_list \n ${module_name} | command: ${command}"
-            echo "Run ${command} failed"
-        fi
+    elif [[ $last_status != 0 ]]; then
+        failed_cmd_list="$failed_cmd_list \n ${module_name} | command: ${command}"
+        echo "Run ${command} failed"
     fi
 }
 
@@ -188,7 +188,9 @@ function run_models(){
                 checker_cmd="${PYTHON_PATH} ${BASE_PATH}/checker.py --check --$check_option --output ${model_output_path} --check_weights_items ${check_weights_items} --module_name ${module_name}"
                 run_command ${checker_cmd} ${module_name}
             done
-            rm -rf ${model_output_path}/*[0-9]*
+            if [[ $last_status -eq 0 ]];then
+                rm -rf ${model_output_path}
+            fi
         fi
     done
     model_dict[$module_name]="$model_list"
@@ -222,16 +224,19 @@ MODULE_OUTPUT_PATH=${BASE_PATH}/outputs
 CONFIG_FILE=${BASE_PATH}/config.txt
 pip config set global.index-url https://mirrors.bfsu.edu.cn/pypi/web/simple
 pip install beautifulsoup4==4.12.3
+pip install tqdm
 pip install markdown
 declare -A weight_dict
 declare -A model_dict
 
 #################################################### 代码风格检查 ######################################################
-pre-commit
-last_status=${PIPESTATUS[0]}
-if [[ $last_status != 0 ]]; then
-    echo "pre-commit check failed, please fix it first."
-    exit 1
+if [[ DEVICE_TYPE == 'gpu' ]];then
+    pre-commit
+    last_status=${PIPESTATUS[0]}
+    if [[ $last_status != 0 ]]; then
+        echo "pre-commit check failed, please fix it first."
+        exit 1
+    fi
 fi
 
 #################################################### 文档超链接检查 ######################################################
@@ -256,9 +261,16 @@ eval $install_pdx_cmd
 if [[ -z $MEM_SIZE ]]; then
     MEM_SIZE=16
 fi
+
 if [[ -z $DEVICE_TYPE ]]; then
     DEVICE_TYPE='gpu'
 fi
+
+PIPE_TYPE=$DEVICE_TYPE
+if [[ $DEVICE_TYPE == 'dcu' ]]; then
+    DEVICE_TYPE='gpu'
+fi
+
 if [[ -z $DEVICE_ID ]]; then
     DEVICE_ID='0,1,2,3'
 fi
@@ -380,6 +392,9 @@ for modules_info in ${modules_info_list[@]}; do
     done
 done
 
+if [[ $PIPE_TYPE != 'gpu' ]];then
+    exit 0
+fi
 #################################################### 产线级测试 ######################################################
 IFS=$'\n'
 PIPELINE_YAML_LIST=`ls paddlex/pipelines | grep .yaml`

diff --git a/models/PaddleX/ci/config.txt b/models/PaddleX/ci/config.txt
@@ -47,8 +47,8 @@ inference_weight_dir:best_accuracy/inference
 epochs_iters:2
 *********************************************************************************************************************
 suite_name:PaddleOCR
-module_name:text_detection_seal
-check_dataset_yaml:paddlex/configs/text_detection_seal/PP-OCRv4_mobile_seal_det.yaml
+module_name:seal_text_detection
+check_dataset_yaml:paddlex/configs/seal_text_detection/PP-OCRv4_mobile_seal_det.yaml
 dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/data/ocr_curve_det_dataset_examples.tar
 train_list_name:train.txt
 run_model:train|evaluate|predict|export
@@ -71,13 +71,25 @@ inference_weight_dir:best_model/inference
 epochs_iters:2
 *********************************************************************************************************************
 suite_name:PaddleDetection
+module_name:rotated_object_detection
+check_dataset_yaml:paddlex/configs/rotated_object_detection/PP-YOLOE-R_L.yaml
+dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/PaddleX3.0/CI/dataset/rdet_dota_examples.tar
+train_list_name:
+run_model:train|evaluate|predict|export
+check_options:check_train_result_json|check_eval_result_json
+check_weights_items:2,pdparams,pdema,pdopt,pdstates,inference_config,pdmodel,pdiparams,pdiparams.info
+evaluate_weight_path:best_model/best_model.pdparams
+inference_weight_dir:best_model/inference
+epochs_iters:2
+*********************************************************************************************************************
+suite_name:PaddleDetection
 module_name:face_detection
 check_dataset_yaml:paddlex/configs/face_detection/PicoDet_LCNet_x2_5_face.yaml
 dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/data/widerface_coco_examples.tar
 train_list_name:
 run_model:train|evaluate|predict|export
 check_options:check_train_result_json|check_eval_result_json
-check_weights_items:2,pdparams,pdema,pdopt,pdstates,inference_config,pdmodel,pdiparams,pdiparams.info
+check_weights_items:2,pdparams,pdopt,pdstates,inference_config,pdmodel,pdiparams,pdiparams.info
 evaluate_weight_path:best_model/best_model.pdparams
 inference_weight_dir:best_model/inference
 epochs_iters:2
@@ -143,8 +155,8 @@ inference_weight_dir:best_model/inference
 epochs_iters:2
 *********************************************************************************************************************
 suite_name:PaddleDetection
-module_name:structure_analysis
-check_dataset_yaml:paddlex/configs/structure_analysis/PicoDet_layout_1x.yaml
+module_name:layout_detection
+check_dataset_yaml:paddlex/configs/layout_detection/PicoDet_layout_1x.yaml
 dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/PaddleX3.0/CI/dataset/det_layout_examples.tar
 train_list_name:
 run_model:train|evaluate|predict|export
@@ -155,8 +167,8 @@ inference_weight_dir:best_model/inference
 epochs_iters:2
 *********************************************************************************************************************
 suite_name:PaddleClas
-module_name:face_recognition
-check_dataset_yaml:paddlex/configs/face_recognition/MobileFaceNet.yaml
+module_name:face_feature
+check_dataset_yaml:paddlex/configs/face_feature/MobileFaceNet.yaml
 dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/data/face_rec_examples.tar
 train_list_name:train.txt
 run_model:train|evaluate|predict|export
@@ -191,8 +203,8 @@ inference_weight_dir:best_model/inference
 epochs_iters:2
 *********************************************************************************************************************
 suite_name:PaddleClas
-module_name:multilabel_classification
-check_dataset_yaml:paddlex/configs/multilabel_classification/ResNet50_ML.yaml
+module_name:image_multilabel_classification
+check_dataset_yaml:paddlex/configs/image_multilabel_classification/ResNet50_ML.yaml
 dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/PaddleX3.0/CI/dataset/mlcls_nus_examples.tar
 train_list_name:train.txt
 run_model:train|evaluate|predict|export
@@ -203,8 +215,8 @@ inference_weight_dir:best_model/inference
 epochs_iters:2
 *********************************************************************************************************************
 suite_name:PaddleClas
-module_name:pedestrian_attribute
-check_dataset_yaml:paddlex/configs/pedestrian_attribute/PP-LCNet_x1_0_pedestrian_attribute.yaml
+module_name:pedestrian_attribute_recognition
+check_dataset_yaml:paddlex/configs/pedestrian_attribute_recognition/PP-LCNet_x1_0_pedestrian_attribute.yaml
 dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/data/pedestrian_attribute_examples.tar
 train_list_name:train.txt
 run_model:train|evaluate|predict|export
@@ -215,8 +227,8 @@ inference_weight_dir:best_model/inference
 epochs_iters:2
 *********************************************************************************************************************
 suite_name:PaddleClas
-module_name:vehicle_attribute
-check_dataset_yaml:paddlex/configs/vehicle_attribute/PP-LCNet_x1_0_vehicle_attribute.yaml
+module_name:vehicle_attribute_recognition
+check_dataset_yaml:paddlex/configs/vehicle_attribute_recognition/PP-LCNet_x1_0_vehicle_attribute.yaml
 dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/data/vehicle_attribute_examples.tar
 train_list_name:train.txt
 run_model:train|evaluate|predict|export
@@ -227,8 +239,20 @@ inference_weight_dir:best_model/inference
 epochs_iters:2
 *********************************************************************************************************************
 suite_name:PaddleClas
-module_name:general_recognition
-check_dataset_yaml:paddlex/configs/general_recognition/PP-ShiTuV2_rec.yaml
+module_name:textline_orientation
+check_dataset_yaml:paddlex/configs/textline_orientation/PP-LCNet_x0_25_textline_ori.yaml
+dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/data/textline_orientation_example_data.tar
+train_list_name:train.txt
+run_model:train|evaluate|predict|export
+check_options:check_train_result_json|check_eval_result_json
+check_weights_items:2,pdparams,pdopt,pdstates,pdmodel,pdiparams,pdiparams.info
+evaluate_weight_path:best_model/best_model.pdparams
+inference_weight_dir:best_model/inference
+epochs_iters:2
+*********************************************************************************************************************
+suite_name:PaddleClas
+module_name:image_feature
+check_dataset_yaml:paddlex/configs/image_feature/PP-ShiTuV2_rec.yaml
 dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/PaddleX3.0/CI/dataset/Inshop_examples.tar
 train_list_name:train.txt
 run_model:train|evaluate|predict|export
@@ -263,8 +287,8 @@ inference_weight_dir:best_model/inference
 epochs_iters:20
 *********************************************************************************************************************
 suite_name:PaddleSeg
-module_name:anomaly_detection
-check_dataset_yaml:paddlex/configs/anomaly_detection/STFPM.yaml
+module_name:image_anomaly_detection
+check_dataset_yaml:paddlex/configs/image_anomaly_detection/STFPM.yaml
 dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/PaddleX3.0/CI/dataset/mvtec_examples.tar
 train_list_name:train.txt
 run_model:train|evaluate|predict|export
@@ -309,3 +333,15 @@ check_weights_items:score,pdparams
 evaluate_weight_path:best_model/model.pdparams
 inference_weight_dir:inference
 epochs_iters:2
+*********************************************************************************************************************
+suite_name:PaddleVideo
+module_name:video_classification
+check_dataset_yaml:paddlex/configs/video_classification/PPTSM_ResNet50_k400_8frames_uniform.yaml
+dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/data/k400_examples.tar
+train_list_name:train.txt
+run_model:train|evaluate|predict|export
+check_options:check_train_result_json|check_eval_result_json
+check_weights_items:2,pdparams,pdopt,pdstates,pdmodel,pdiparams,pdiparams.info
+evaluate_weight_path:best_model/best_model.pdparams
+inference_weight_dir:best_model/inference
+epochs_iters:2