From 106948094cc354f94bb3268cf25c5f601a17b58c Mon Sep 17 00:00:00 2001 From: LixinGuo <18510030324@126.com> Date: Fri, 27 Sep 2024 15:24:41 +0800 Subject: [PATCH] remove api cases not suport in pir mode,test=model (#2960) * set FLAGS_use_stride_kernel true * update dnn threshold * update dnn threshold,test=model * test=model,update base_value because of Paddle PR##53584 * fix code_style and pandas=1.5.3,test=model * fix dy2st run error * fix dy2st run error,test=model * fix rec, test=model * test=model,fix xly list * add PTS list;test=model * add PTS list;test=model * fix step name;test=model * fix step name;test=model * fix step name;test=model * fix step name;test=model * fix step name;test=model * fix step name;test=model * fix step name;test=model * remove cuda102 py37 * Update dssm base value due to cuda102 to cuda112 * Update models^contentunderstanding^textcnn^config_bigdata.yaml * Update PaddleRec_XLY_list * add paddlerec ce, test=model * fix bert train dir,test=model * fix distributed api error * fix nlp pip error,test=model * remove api cases not suport in pir mode --- distributed/CE_API/run_gpu/start.sh | 1 - distributed/CE_API/test/test_dist_auto_api.py | 126 +++++++++--------- .../test_dist_collective_communicator_api.py | 20 +-- .../CE_API/test/test_dist_fleet_init.py | 20 +-- 4 files changed, 83 insertions(+), 84 deletions(-) diff --git a/distributed/CE_API/run_gpu/start.sh b/distributed/CE_API/run_gpu/start.sh index cb1ba94d6c..c97dff923c 100644 --- a/distributed/CE_API/run_gpu/start.sh +++ b/distributed/CE_API/run_gpu/start.sh @@ -41,7 +41,6 @@ cases="test_dist_auto_api.py \ test_dist_shard_tensor.py \ test_dist_shard.py \ test_dist_showclickentry.py \ - test_dist_split.py \ test_dist_strategy.py \ test_dist_unshard_dtensor.py \ test_dist_utils_recompute.py \ diff --git a/distributed/CE_API/test/test_dist_auto_api.py b/distributed/CE_API/test/test_dist_auto_api.py index 6ea5b57933..30e1485c31 100755 --- a/distributed/CE_API/test/test_dist_auto_api.py +++ b/distributed/CE_API/test/test_dist_auto_api.py @@ -22,51 +22,51 @@ class TestDistAutoApi(object): """TestDistAutoApi""" - def test_auto_process_mesh(self): - """test_auto_process_mesh""" - cmd = "python -m paddle.distributed.launch --devices 0,1 --job_id auto_process_mesh dist_auto_process_mesh.py" - pro = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = pro.communicate() - print(out) - pro.wait() - pro.returncode == 0 - assert str(out).find("Error") == -1 - assert str(err).find("Error") == -1 + # def test_auto_process_mesh(self): + # """test_auto_process_mesh""" + # cmd = "python -m paddle.distributed.launch --devices 0,1 --job_id auto_process_mesh dist_auto_process_mesh.py" + # pro = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # out, err = pro.communicate() + # print(out) + # pro.wait() + # pro.returncode == 0 + # assert str(out).find("Error") == -1 + # assert str(err).find("Error") == -1 - def test_auto_process_mesh_v2(self): - """test_auto_process_mesh""" - cmd = ( - "python -m paddle.distributed.launch --devices 0,1 --job_id auto_process_mesh dist_auto_process_mesh_v2.py" - ) - pro = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = pro.communicate() - print(out) - pro.wait() - pro.returncode == 0 - assert str(out).find("Error") == -1 - assert str(err).find("Error") == -1 + # def test_auto_process_mesh_v2(self): + # """test_auto_process_mesh""" + # cmd = ( + # "python -m paddle.distributed.launch --devices 0,1 --job_id auto_process_mesh dist_auto_process_mesh_v2.py" + # ) + # pro = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # out, err = pro.communicate() + # print(out) + # pro.wait() + # pro.returncode == 0 + # assert str(out).find("Error") == -1 + # assert str(err).find("Error") == -1 - def test_auto_shard_op(self): - """test_auto_shard_op""" - cmd = "python -m paddle.distributed.launch --devices 0,1 --job_id auto_shard_op dist_auto_shard_op.py" - pro = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = pro.communicate() - print(out) - pro.wait() - pro.returncode == 0 - assert str(out).find("Error") == -1 - assert str(err).find("Error") == -1 + # def test_auto_shard_op(self): + # """test_auto_shard_op""" + # cmd = "python -m paddle.distributed.launch --devices 0,1 --job_id auto_shard_op dist_auto_shard_op.py" + # pro = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # out, err = pro.communicate() + # print(out) + # pro.wait() + # pro.returncode == 0 + # assert str(out).find("Error") == -1 + # assert str(err).find("Error") == -1 - def test_auto_shard_tensor(self): - """test_auto_shard_tensor""" - cmd = "python -m paddle.distributed.launch --devices 0,1 --job_id auto_shard_tensor dist_auto_shard_tensor.py" - pro = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = pro.communicate() - print(out) - pro.wait() - pro.returncode == 0 - assert str(out).find("Error") == -1 - assert str(err).find("Error") == -1 + # def test_auto_shard_tensor(self): + # """test_auto_shard_tensor""" + # cmd = "python -m paddle.distributed.launch --devices 0,1 --job_id auto_shard_tensor dist_auto_shard_tensor.py" + # pro = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # out, err = pro.communicate() + # print(out) + # pro.wait() + # pro.returncode == 0 + # assert str(out).find("Error") == -1 + # assert str(err).find("Error") == -1 def test_auto_converter(self): """test_auto_converter""" @@ -79,17 +79,17 @@ def test_auto_converter(self): assert str(out).find("Error") == -1 assert str(err).find("Error") == -1 - def test_auto_new_cost_model(self): - """test_auto_new_cost_model""" - cmd = "python -m paddle.distributed.launch --devices 0,1 --job_id auto_new_cost_model \ - dist_auto_new_cost_model.py" - pro = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = pro.communicate() - print(out) - pro.wait() - pro.returncode == 0 - assert str(out).find("Error") == -1 - assert str(err).find("Error") == -1 + # def test_auto_new_cost_model(self): + # """test_auto_new_cost_model""" + # cmd = "python -m paddle.distributed.launch --devices 0,1 --job_id auto_new_cost_model \ + # dist_auto_new_cost_model.py" + # pro = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # out, err = pro.communicate() + # print(out) + # pro.wait() + # pro.returncode == 0 + # assert str(out).find("Error") == -1 + # assert str(err).find("Error") == -1 def test_auto_comm_cost(self): """test_auto_comm_cost""" @@ -113,13 +113,13 @@ def test_auto_comp_cost(self): assert str(out).find("Error") == -1 assert str(err).find("Error") == -1 - def test_auto_base_cost(self): - """test_auto_base_cost""" - cmd = "python -m paddle.distributed.launch --devices 0,1 --job_id auto_base_cost dist_auto_base_cost.py" - pro = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = pro.communicate() - print(out) - pro.wait() - pro.returncode == 0 - assert str(out).find("Error") == -1 - assert str(err).find("Error") == -1 + # def test_auto_base_cost(self): + # """test_auto_base_cost""" + # cmd = "python -m paddle.distributed.launch --devices 0,1 --job_id auto_base_cost dist_auto_base_cost.py" + # pro = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # out, err = pro.communicate() + # print(out) + # pro.wait() + # pro.returncode == 0 + # assert str(out).find("Error") == -1 + # assert str(err).find("Error") == -1 diff --git a/distributed/CE_API/test/test_dist_collective_communicator_api.py b/distributed/CE_API/test/test_dist_collective_communicator_api.py index bb7f74fc48..62929a5bd5 100755 --- a/distributed/CE_API/test/test_dist_collective_communicator_api.py +++ b/distributed/CE_API/test/test_dist_collective_communicator_api.py @@ -141,16 +141,16 @@ def test_collective_reduce_scatter(self): assert str(out).find("Error") == -1 assert str(err).find("Error") == -1 - def test_collective_split(self): - """test_collective_split""" - cmd = "python -m paddle.distributed.launch --devices 0,1 --job_id split dist_collective_split.py" - pro = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = pro.communicate() - print(out) - pro.wait() - pro.returncode == 0 - assert str(out).find("Error") == -1 - assert str(err).find("Error") == -1 + # def test_collective_split(self): + # """test_collective_split""" + # cmd = "python -m paddle.distributed.launch --devices 0,1 --job_id split dist_collective_split.py" + # pro = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # out, err = pro.communicate() + # print(out) + # pro.wait() + # pro.returncode == 0 + # assert str(out).find("Error") == -1 + # assert str(err).find("Error") == -1 def test_collective_send(self): """test_collective_send""" diff --git a/distributed/CE_API/test/test_dist_fleet_init.py b/distributed/CE_API/test/test_dist_fleet_init.py index 10f9b9cfb8..9439598be2 100755 --- a/distributed/CE_API/test/test_dist_fleet_init.py +++ b/distributed/CE_API/test/test_dist_fleet_init.py @@ -68,13 +68,13 @@ def test_dist_fleet_init_strategy(self): assert str(out).find("Error") == -1 assert str(err).find("Error") == -1 - def test_dist_fleet_qat_init(self): - """test_dist_fleet_qat_init""" - cmd = "python -m paddle.distributed.launch --devices 0,1 --job_id qat_init \ - dist_fleet_qat_init.py" - pro = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = pro.communicate() - pro.wait() - pro.returncode == 0 - assert str(out).find("Error") == -1 - assert str(err).find("Error") == -1 + # def test_dist_fleet_qat_init(self): + # """test_dist_fleet_qat_init""" + # cmd = "python -m paddle.distributed.launch --devices 0,1 --job_id qat_init \ + # dist_fleet_qat_init.py" + # pro = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # out, err = pro.communicate() + # pro.wait() + # pro.returncode == 0 + # assert str(out).find("Error") == -1 + # assert str(err).find("Error") == -1