From 7815d61872ca5f124f706621853426bb02a35a05 Mon Sep 17 00:00:00 2001 From: changdazhou <142379845+changdazhou@users.noreply.github.com> Date: Wed, 27 Nov 2024 10:21:46 +0800 Subject: [PATCH] update paddlex ci (#2990) --- models/PaddleX/ci/black_list.txt | 27 +----------- models/PaddleX/ci/checker.py | 7 ++-- models/PaddleX/ci/ci_run.sh | 47 +++++++++++++-------- models/PaddleX/ci/config.txt | 70 ++++++++++++++++++++++++-------- 4 files changed, 89 insertions(+), 62 deletions(-) diff --git a/models/PaddleX/ci/black_list.txt b/models/PaddleX/ci/black_list.txt index 61eee9a96b..3b4974229f 100644 --- a/models/PaddleX/ci/black_list.txt +++ b/models/PaddleX/ci/black_list.txt @@ -1,36 +1,11 @@ ################################### All ################################## -All:PP-YOLOE_plus_SOD-largesize-L ################################## Train ################################# ################################ Evaluate ################################ ################################# Predict ################################ -Predict:PP-YOLOE_seg-S -Predict:MaskRCNN-ResNet50 -Predict:MaskRCNN-ResNet50-FPN -Predict:MaskRCNN-ResNet50-vd-FPN -Predict:MaskRCNN-ResNet101-FPN -Predict:MaskRCNN-ResNet101-vd-FPN -Predict:MaskRCNN-ResNeXt101-vd-FPN -Predict:Cascade-MaskRCNN-ResNet50-vd-SSLDv2-FPN -Predict:Cascade-MaskRCNN-ResNet50-FPN -Predict:Nonstationary_ad Predict:TimesNet_ad -Predict:PatchTST_ad -Predict:DLinear_ad -Predict:PP-YOLOE_plus_SOD-L -Predict:PP-YOLOE_plus_SOD-S -Predict:Cascade-FasterRCNN-ResNet50-vd-SSLDv2-FPN -Predict:Cascade-FasterRCNN-ResNet50-FPN -Predict:FasterRCNN-Swin-Tiny-FPN -Predict:FasterRCNN-ResNeXt101-vd-FPN -Predict:FasterRCNN-ResNet101-FPN -Predict:FasterRCNN-ResNet101 -Predict:FasterRCNN-ResNet50-vd-SSLDv2-FPN -Predict:FasterRCNN-ResNet50-vd-FPN -Predict:FasterRCNN-ResNet50-FPN -Predict:FasterRCNN-ResNet50 -Predict:FasterRCNN-ResNet34-FPN ################################## Export ################################ ################################# Pipeline ############################### Pipeline:PP-ChatOCRv3-doc Pipeline:PP-ShiTuV2 Pipeline:face_recognition +Pipeline:ts_ad diff --git a/models/PaddleX/ci/checker.py b/models/PaddleX/ci/checker.py index 1ab9a6d729..ae1e779fbe 100644 --- a/models/PaddleX/ci/checker.py +++ b/models/PaddleX/ci/checker.py @@ -227,8 +227,8 @@ def check_dataset_json_content(self, output_dir, module_name, dataset_result_jso pass_flag = False else: show_type = json_data.get("show_type") - if show_type not in ["image", "txt", "csv"]: - check_dataset_json_message.append(f"检查失败：{show_type} 必须为'image', 'txt', 'csv'其中一个") + if show_type not in ["image", "txt", "csv", "video"]: + check_dataset_json_message.append(f"检查失败：{show_type} 必须为'image', 'txt', 'csv','video'其中一个") pass_flag = False if module_name == "general_recognition": @@ -249,6 +249,7 @@ def check_dataset_json_content(self, output_dir, module_name, dataset_result_jso or "table" in module_name or "formula" in module_name or module_name == "general_recognition" + or module_name == "face_feature" ): try: num_class = int(json_data["attributes"].get("num_classes")) @@ -368,7 +369,7 @@ def run_checks(self, args): if args.check_train_result_json: # 检查 train_result.json 内容 - train_result_json = os.path.join(output_dir, "train_results.json") + train_result_json = os.path.join(output_dir, "train_result.json") check_weights_items = args.check_weights_items check_train_json_message = [] check_train_json_flag, check_train_json_message = self.check_train_json_content( diff --git a/models/PaddleX/ci/ci_run.sh b/models/PaddleX/ci/ci_run.sh index 85129a991d..aea3eac930 100644 --- a/models/PaddleX/ci/ci_run.sh +++ b/models/PaddleX/ci/ci_run.sh @@ -11,8 +11,8 @@ MODEL_LIST_FILE=$2 # MD_NUM: PR中MD文件改动数量，用于判断是否需要进行文档超链接检测，默认为空，设置示例：export MD_NUM=10 # WITHOUT_MD_NUM: PR中MD文件改动之外的改动文件数量，用于判断进行文档超链接检测后是否进行正常的CI，默认为空，设置示例：export WITHOUT_MD_NUM=10 -# set -x if [[ $MODE == 'PaddleX' ]];then + set -x failed_cmd_list="" fi @@ -52,25 +52,25 @@ function run_command(){ printf "\e[32m|%-20s| %-50s | %-20s\n\e[0m" "[${time_stamp}]" "${command}" eval $command last_status=${PIPESTATUS[0]} + n=1 + # Try 2 times to run command if it fails if [[ $MODE != 'PaddleX' ]];then - n=1 - # Try 3 times to run command if it fails while [[ $last_status != 0 ]]; do sleep 10 n=`expr $n + 1` printf "\e[32m|%-20s| %-50s | %-20s\n\e[0m" "[${time_stamp}]" "${command}" + sync + echo 1 > /proc/sys/vm/drop_caches eval $command last_status=${PIPESTATUS[0]} - if [[ $n -eq 3 && $last_status != 0 ]]; then - echo "Retry 3 times failed with command: ${command}" + if [[ $n -eq 2 && $last_status != 0 ]]; then + echo "Retry 2 times failed with command: ${command}" exit 1 fi done - else - if [[ $last_status != 0 ]];then - failed_cmd_list="$failed_cmd_list \n ${module_name} | command: ${command}" - echo "Run ${command} failed" - fi + elif [[ $last_status != 0 ]]; then + failed_cmd_list="$failed_cmd_list \n ${module_name} | command: ${command}" + echo "Run ${command} failed" fi } @@ -188,7 +188,9 @@ function run_models(){ checker_cmd="${PYTHON_PATH} ${BASE_PATH}/checker.py --check --$check_option --output ${model_output_path} --check_weights_items ${check_weights_items} --module_name ${module_name}" run_command ${checker_cmd} ${module_name} done - rm -rf ${model_output_path}/*[0-9]* + if [[ $last_status -eq 0 ]];then + rm -rf ${model_output_path} + fi fi done model_dict[$module_name]="$model_list" @@ -222,16 +224,19 @@ MODULE_OUTPUT_PATH=${BASE_PATH}/outputs CONFIG_FILE=${BASE_PATH}/config.txt pip config set global.index-url https://mirrors.bfsu.edu.cn/pypi/web/simple pip install beautifulsoup4==4.12.3 +pip install tqdm pip install markdown declare -A weight_dict declare -A model_dict #################################################### 代码风格检查 ###################################################### -pre-commit -last_status=${PIPESTATUS[0]} -if [[ $last_status != 0 ]]; then - echo "pre-commit check failed, please fix it first." - exit 1 +if [[ DEVICE_TYPE == 'gpu' ]];then + pre-commit + last_status=${PIPESTATUS[0]} + if [[ $last_status != 0 ]]; then + echo "pre-commit check failed, please fix it first." + exit 1 + fi fi #################################################### 文档超链接检查 ###################################################### @@ -256,9 +261,16 @@ eval $install_pdx_cmd if [[ -z $MEM_SIZE ]]; then MEM_SIZE=16 fi + if [[ -z $DEVICE_TYPE ]]; then DEVICE_TYPE='gpu' fi + +PIPE_TYPE=$DEVICE_TYPE +if [[ $DEVICE_TYPE == 'dcu' ]]; then + DEVICE_TYPE='gpu' +fi + if [[ -z $DEVICE_ID ]]; then DEVICE_ID='0,1,2,3' fi @@ -380,6 +392,9 @@ for modules_info in ${modules_info_list[@]}; do done done +if [[ $PIPE_TYPE != 'gpu' ]];then + exit 0 +fi #################################################### 产线级测试 ###################################################### IFS=$'\n' PIPELINE_YAML_LIST=`ls paddlex/pipelines | grep .yaml` diff --git a/models/PaddleX/ci/config.txt b/models/PaddleX/ci/config.txt index 713bf529f0..a6b3414bbb 100644 --- a/models/PaddleX/ci/config.txt +++ b/models/PaddleX/ci/config.txt @@ -47,8 +47,8 @@ inference_weight_dir:best_accuracy/inference epochs_iters:2 ********************************************************************************************************************* suite_name:PaddleOCR -module_name:text_detection_seal -check_dataset_yaml:paddlex/configs/text_detection_seal/PP-OCRv4_mobile_seal_det.yaml +module_name:seal_text_detection +check_dataset_yaml:paddlex/configs/seal_text_detection/PP-OCRv4_mobile_seal_det.yaml dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/data/ocr_curve_det_dataset_examples.tar train_list_name:train.txt run_model:train|evaluate|predict|export @@ -71,13 +71,25 @@ inference_weight_dir:best_model/inference epochs_iters:2 ********************************************************************************************************************* suite_name:PaddleDetection +module_name:rotated_object_detection +check_dataset_yaml:paddlex/configs/rotated_object_detection/PP-YOLOE-R_L.yaml +dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/PaddleX3.0/CI/dataset/rdet_dota_examples.tar +train_list_name: +run_model:train|evaluate|predict|export +check_options:check_train_result_json|check_eval_result_json +check_weights_items:2,pdparams,pdema,pdopt,pdstates,inference_config,pdmodel,pdiparams,pdiparams.info +evaluate_weight_path:best_model/best_model.pdparams +inference_weight_dir:best_model/inference +epochs_iters:2 +********************************************************************************************************************* +suite_name:PaddleDetection module_name:face_detection check_dataset_yaml:paddlex/configs/face_detection/PicoDet_LCNet_x2_5_face.yaml dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/data/widerface_coco_examples.tar train_list_name: run_model:train|evaluate|predict|export check_options:check_train_result_json|check_eval_result_json -check_weights_items:2,pdparams,pdema,pdopt,pdstates,inference_config,pdmodel,pdiparams,pdiparams.info +check_weights_items:2,pdparams,pdopt,pdstates,inference_config,pdmodel,pdiparams,pdiparams.info evaluate_weight_path:best_model/best_model.pdparams inference_weight_dir:best_model/inference epochs_iters:2 @@ -143,8 +155,8 @@ inference_weight_dir:best_model/inference epochs_iters:2 ********************************************************************************************************************* suite_name:PaddleDetection -module_name:structure_analysis -check_dataset_yaml:paddlex/configs/structure_analysis/PicoDet_layout_1x.yaml +module_name:layout_detection +check_dataset_yaml:paddlex/configs/layout_detection/PicoDet_layout_1x.yaml dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/PaddleX3.0/CI/dataset/det_layout_examples.tar train_list_name: run_model:train|evaluate|predict|export @@ -155,8 +167,8 @@ inference_weight_dir:best_model/inference epochs_iters:2 ********************************************************************************************************************* suite_name:PaddleClas -module_name:face_recognition -check_dataset_yaml:paddlex/configs/face_recognition/MobileFaceNet.yaml +module_name:face_feature +check_dataset_yaml:paddlex/configs/face_feature/MobileFaceNet.yaml dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/data/face_rec_examples.tar train_list_name:train.txt run_model:train|evaluate|predict|export @@ -191,8 +203,8 @@ inference_weight_dir:best_model/inference epochs_iters:2 ********************************************************************************************************************* suite_name:PaddleClas -module_name:multilabel_classification -check_dataset_yaml:paddlex/configs/multilabel_classification/ResNet50_ML.yaml +module_name:image_multilabel_classification +check_dataset_yaml:paddlex/configs/image_multilabel_classification/ResNet50_ML.yaml dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/PaddleX3.0/CI/dataset/mlcls_nus_examples.tar train_list_name:train.txt run_model:train|evaluate|predict|export @@ -203,8 +215,8 @@ inference_weight_dir:best_model/inference epochs_iters:2 ********************************************************************************************************************* suite_name:PaddleClas -module_name:pedestrian_attribute -check_dataset_yaml:paddlex/configs/pedestrian_attribute/PP-LCNet_x1_0_pedestrian_attribute.yaml +module_name:pedestrian_attribute_recognition +check_dataset_yaml:paddlex/configs/pedestrian_attribute_recognition/PP-LCNet_x1_0_pedestrian_attribute.yaml dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/data/pedestrian_attribute_examples.tar train_list_name:train.txt run_model:train|evaluate|predict|export @@ -215,8 +227,8 @@ inference_weight_dir:best_model/inference epochs_iters:2 ********************************************************************************************************************* suite_name:PaddleClas -module_name:vehicle_attribute -check_dataset_yaml:paddlex/configs/vehicle_attribute/PP-LCNet_x1_0_vehicle_attribute.yaml +module_name:vehicle_attribute_recognition +check_dataset_yaml:paddlex/configs/vehicle_attribute_recognition/PP-LCNet_x1_0_vehicle_attribute.yaml dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/data/vehicle_attribute_examples.tar train_list_name:train.txt run_model:train|evaluate|predict|export @@ -227,8 +239,20 @@ inference_weight_dir:best_model/inference epochs_iters:2 ********************************************************************************************************************* suite_name:PaddleClas -module_name:general_recognition -check_dataset_yaml:paddlex/configs/general_recognition/PP-ShiTuV2_rec.yaml +module_name:textline_orientation +check_dataset_yaml:paddlex/configs/textline_orientation/PP-LCNet_x0_25_textline_ori.yaml +dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/data/textline_orientation_example_data.tar +train_list_name:train.txt +run_model:train|evaluate|predict|export +check_options:check_train_result_json|check_eval_result_json +check_weights_items:2,pdparams,pdopt,pdstates,pdmodel,pdiparams,pdiparams.info +evaluate_weight_path:best_model/best_model.pdparams +inference_weight_dir:best_model/inference +epochs_iters:2 +********************************************************************************************************************* +suite_name:PaddleClas +module_name:image_feature +check_dataset_yaml:paddlex/configs/image_feature/PP-ShiTuV2_rec.yaml dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/PaddleX3.0/CI/dataset/Inshop_examples.tar train_list_name:train.txt run_model:train|evaluate|predict|export @@ -263,8 +287,8 @@ inference_weight_dir:best_model/inference epochs_iters:20 ********************************************************************************************************************* suite_name:PaddleSeg -module_name:anomaly_detection -check_dataset_yaml:paddlex/configs/anomaly_detection/STFPM.yaml +module_name:image_anomaly_detection +check_dataset_yaml:paddlex/configs/image_anomaly_detection/STFPM.yaml dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/PaddleX3.0/CI/dataset/mvtec_examples.tar train_list_name:train.txt run_model:train|evaluate|predict|export @@ -309,3 +333,15 @@ check_weights_items:score,pdparams evaluate_weight_path:best_model/model.pdparams inference_weight_dir:inference epochs_iters:2 +********************************************************************************************************************* +suite_name:PaddleVideo +module_name:video_classification +check_dataset_yaml:paddlex/configs/video_classification/PPTSM_ResNet50_k400_8frames_uniform.yaml +dataset_url:https://paddle-model-ecology.bj.bcebos.com/paddlex/data/k400_examples.tar +train_list_name:train.txt +run_model:train|evaluate|predict|export +check_options:check_train_result_json|check_eval_result_json +check_weights_items:2,pdparams,pdopt,pdstates,pdmodel,pdiparams,pdiparams.info +evaluate_weight_path:best_model/best_model.pdparams +inference_weight_dir:best_model/inference +epochs_iters:2