diff --git a/benchmark/__init__.py b/benchmark/__init__.py index d4a5f1a2..4e64445e 100644 --- a/benchmark/__init__.py +++ b/benchmark/__init__.py @@ -1,4 +1,6 @@ import numpy as np +import os +import unittest import oneflow as flow diff --git a/benchmark/test_alexnet.py b/benchmark/test_alexnet.py index a6d8984d..da614d4c 100644 --- a/benchmark/test_alexnet.py +++ b/benchmark/test_alexnet.py @@ -3,30 +3,50 @@ from flowvision.models.alexnet import alexnet +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_alexnet_batch_size1(benchmark, net=alexnet, input_shape=[1, 3, 224, 224]): model, x, optimizer = fetch_args(net, input_shape) benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_alexnet_batch_size2(benchmark, net=alexnet, input_shape=[2, 3, 224, 224]): model, x, optimizer = fetch_args(net, input_shape) benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_alexnet_batch_size4(benchmark, net=alexnet, input_shape=[4, 3, 224, 224]): model, x, optimizer = fetch_args(net, input_shape) benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_alexnet_batch_size8(benchmark, net=alexnet, input_shape=[8, 3, 224, 224]): model, x, optimizer = fetch_args(net, input_shape) benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_alexnet_batch_size16(benchmark, net=alexnet, input_shape=[16, 3, 224, 224]): model, x, optimizer = fetch_args(net, input_shape) diff --git a/benchmark/test_convnext.py b/benchmark/test_convnext.py index e671eabf..488a4e4d 100644 --- a/benchmark/test_convnext.py +++ b/benchmark/test_convnext.py @@ -3,6 +3,10 @@ from flowvision.models.convnext import convnext_tiny_224 +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_convnext_tiny_224_batch_size1( benchmark, net=convnext_tiny_224, input_shape=[1, 3, 224, 224] @@ -11,6 +15,10 @@ def test_convnext_tiny_224_batch_size1( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_convnext_tiny_224_batch_size2( benchmark, net=convnext_tiny_224, input_shape=[2, 3, 224, 224] @@ -19,6 +27,10 @@ def test_convnext_tiny_224_batch_size2( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_convnext_tiny_224_batch_size4( benchmark, net=convnext_tiny_224, input_shape=[4, 3, 224, 224] @@ -27,6 +39,10 @@ def test_convnext_tiny_224_batch_size4( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_convnext_tiny_224_batch_size8( benchmark, net=convnext_tiny_224, input_shape=[8, 3, 224, 224] @@ -35,6 +51,10 @@ def test_convnext_tiny_224_batch_size8( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_convnext_tiny_224_batch_size16( benchmark, net=convnext_tiny_224, input_shape=[16, 3, 224, 224] diff --git a/benchmark/test_crossformer.py b/benchmark/test_crossformer.py index e9bbb7ef..5de651ea 100644 --- a/benchmark/test_crossformer.py +++ b/benchmark/test_crossformer.py @@ -3,6 +3,10 @@ from flowvision.models.crossformer import crossformer_tiny_patch4_group7_224 +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_crossformer_tiny_patch4_group7_224_batch_size1( benchmark, net=crossformer_tiny_patch4_group7_224, input_shape=[1, 3, 224, 224] @@ -11,6 +15,10 @@ def test_crossformer_tiny_patch4_group7_224_batch_size1( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_crossformer_tiny_patch4_group7_224_batch_size2( benchmark, net=crossformer_tiny_patch4_group7_224, input_shape=[2, 3, 224, 224] @@ -19,6 +27,10 @@ def test_crossformer_tiny_patch4_group7_224_batch_size2( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_crossformer_tiny_patch4_group7_224_batch_size4( benchmark, net=crossformer_tiny_patch4_group7_224, input_shape=[4, 3, 224, 224] @@ -27,6 +39,10 @@ def test_crossformer_tiny_patch4_group7_224_batch_size4( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_crossformer_tiny_patch4_group7_224_batch_size8( benchmark, net=crossformer_tiny_patch4_group7_224, input_shape=[8, 3, 224, 224] @@ -35,6 +51,10 @@ def test_crossformer_tiny_patch4_group7_224_batch_size8( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_crossformer_tiny_patch4_group7_224_batch_size16( benchmark, net=crossformer_tiny_patch4_group7_224, input_shape=[16, 3, 224, 224] diff --git a/benchmark/test_cswin.py b/benchmark/test_cswin.py index 2b4cf8ef..8742a372 100644 --- a/benchmark/test_cswin.py +++ b/benchmark/test_cswin.py @@ -3,6 +3,10 @@ from flowvision.models.cswin import cswin_tiny_224 +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_cswin_tiny_224_batch_size1( benchmark, net=cswin_tiny_224, input_shape=[1, 3, 224, 224] @@ -11,6 +15,10 @@ def test_cswin_tiny_224_batch_size1( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_cswin_tiny_224_batch_size2( benchmark, net=cswin_tiny_224, input_shape=[2, 3, 224, 224] @@ -19,6 +27,10 @@ def test_cswin_tiny_224_batch_size2( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_cswin_tiny_224_batch_size4( benchmark, net=cswin_tiny_224, input_shape=[4, 3, 224, 224] @@ -27,6 +39,10 @@ def test_cswin_tiny_224_batch_size4( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_cswin_tiny_224_batch_size8( benchmark, net=cswin_tiny_224, input_shape=[8, 3, 224, 224] @@ -35,6 +51,10 @@ def test_cswin_tiny_224_batch_size8( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_cswin_tiny_224_batch_size16( benchmark, net=cswin_tiny_224, input_shape=[16, 3, 224, 224] diff --git a/benchmark/test_densenet.py b/benchmark/test_densenet.py index a95bbae4..68fcbfa1 100644 --- a/benchmark/test_densenet.py +++ b/benchmark/test_densenet.py @@ -3,6 +3,10 @@ from flowvision.models.densenet import densenet121 +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_densenet121_batch_size1( benchmark, net=densenet121, input_shape=[1, 3, 224, 224] diff --git a/benchmark/test_ghostnet.py b/benchmark/test_ghostnet.py index 805c481d..e2373b55 100644 --- a/benchmark/test_ghostnet.py +++ b/benchmark/test_ghostnet.py @@ -3,30 +3,50 @@ from flowvision.models.ghostnet import ghostnet +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_ghostnet_batch_size1(benchmark, net=ghostnet, input_shape=[1, 3, 224, 224]): model, x, optimizer = fetch_args(net, input_shape) benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_ghostnet_batch_size2(benchmark, net=ghostnet, input_shape=[2, 3, 224, 224]): model, x, optimizer = fetch_args(net, input_shape) benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_ghostnet_batch_size4(benchmark, net=ghostnet, input_shape=[4, 3, 224, 224]): model, x, optimizer = fetch_args(net, input_shape) benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_ghostnet_batch_size8(benchmark, net=ghostnet, input_shape=[8, 3, 224, 224]): model, x, optimizer = fetch_args(net, input_shape) benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_ghostnet_batch_size16(benchmark, net=ghostnet, input_shape=[16, 3, 224, 224]): model, x, optimizer = fetch_args(net, input_shape) diff --git a/benchmark/test_mlp_mixer.py b/benchmark/test_mlp_mixer.py index b5f833e7..bfea08b3 100644 --- a/benchmark/test_mlp_mixer.py +++ b/benchmark/test_mlp_mixer.py @@ -3,6 +3,10 @@ from flowvision.models.mlp_mixer import mlp_mixer_b16_224 +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_mlp_mixer_b16_224_batch_size1( benchmark, net=mlp_mixer_b16_224, input_shape=[1, 3, 224, 224] @@ -11,6 +15,10 @@ def test_mlp_mixer_b16_224_batch_size1( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_mlp_mixer_b16_224_batch_size2( benchmark, net=mlp_mixer_b16_224, input_shape=[2, 3, 224, 224] @@ -19,6 +27,10 @@ def test_mlp_mixer_b16_224_batch_size2( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_mlp_mixer_b16_224_batch_size4( benchmark, net=mlp_mixer_b16_224, input_shape=[4, 3, 224, 224] @@ -27,6 +39,10 @@ def test_mlp_mixer_b16_224_batch_size4( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_mlp_mixer_b16_224_batch_size8( benchmark, net=mlp_mixer_b16_224, input_shape=[8, 3, 224, 224] @@ -35,6 +51,10 @@ def test_mlp_mixer_b16_224_batch_size8( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_mlp_mixer_b16_224_batch_size16( benchmark, net=mlp_mixer_b16_224, input_shape=[16, 3, 224, 224] diff --git a/benchmark/test_mnasnet.py b/benchmark/test_mnasnet.py index 132e49b2..00ef3128 100644 --- a/benchmark/test_mnasnet.py +++ b/benchmark/test_mnasnet.py @@ -3,6 +3,10 @@ from flowvision.models.mnasnet import mnasnet0_5 +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_mnasnet0_5_batch_size1( benchmark, net=mnasnet0_5, input_shape=[1, 3, 224, 224] @@ -11,6 +15,10 @@ def test_mnasnet0_5_batch_size1( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_mnasnet0_5_batch_size2( benchmark, net=mnasnet0_5, input_shape=[2, 3, 224, 224] @@ -19,6 +27,10 @@ def test_mnasnet0_5_batch_size2( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_mnasnet0_5_batch_size4( benchmark, net=mnasnet0_5, input_shape=[4, 3, 224, 224] @@ -27,6 +39,10 @@ def test_mnasnet0_5_batch_size4( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_mnasnet0_5_batch_size8( benchmark, net=mnasnet0_5, input_shape=[8, 3, 224, 224] @@ -35,6 +51,10 @@ def test_mnasnet0_5_batch_size8( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_mnasnet0_5_batch_size16( benchmark, net=mnasnet0_5, input_shape=[16, 3, 224, 224] diff --git a/benchmark/test_mobilenet.py b/benchmark/test_mobilenet.py index 0b0b6cf6..d00e40bf 100644 --- a/benchmark/test_mobilenet.py +++ b/benchmark/test_mobilenet.py @@ -44,6 +44,10 @@ def test_mobilenet_v3_large_batch_size16( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_mobilenet_v2_batch_size1( benchmark, net=mobilenet_v2, input_shape=[1, 3, 224, 224] diff --git a/benchmark/test_pvt.py b/benchmark/test_pvt.py index 0f758132..2cf37322 100644 --- a/benchmark/test_pvt.py +++ b/benchmark/test_pvt.py @@ -4,60 +4,100 @@ from flowvision.models.pvt import pvt_small +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_pvt_tiny_batch_size1(benchmark, net=pvt_tiny, input_shape=[1, 3, 224, 224]): model, x, optimizer = fetch_args(net, input_shape) benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_pvt_tiny_batch_size2(benchmark, net=pvt_tiny, input_shape=[2, 3, 224, 224]): model, x, optimizer = fetch_args(net, input_shape) benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_pvt_tiny_batch_size4(benchmark, net=pvt_tiny, input_shape=[4, 3, 224, 224]): model, x, optimizer = fetch_args(net, input_shape) benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_pvt_tiny_batch_size8(benchmark, net=pvt_tiny, input_shape=[8, 3, 224, 224]): model, x, optimizer = fetch_args(net, input_shape) benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_pvt_tiny_batch_size16(benchmark, net=pvt_tiny, input_shape=[16, 3, 224, 224]): model, x, optimizer = fetch_args(net, input_shape) benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_pvt_small_batch_size1(benchmark, net=pvt_small, input_shape=[1, 3, 224, 224]): model, x, optimizer = fetch_args(net, input_shape) benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_pvt_small_batch_size2(benchmark, net=pvt_small, input_shape=[2, 3, 224, 224]): model, x, optimizer = fetch_args(net, input_shape) benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_pvt_small_batch_size4(benchmark, net=pvt_small, input_shape=[4, 3, 224, 224]): model, x, optimizer = fetch_args(net, input_shape) benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_pvt_small_batch_size8(benchmark, net=pvt_small, input_shape=[8, 3, 224, 224]): model, x, optimizer = fetch_args(net, input_shape) benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_pvt_small_batch_size16( benchmark, net=pvt_small, input_shape=[16, 3, 224, 224] diff --git a/benchmark/test_res2net.py b/benchmark/test_res2net.py index 35a75bc1..b754e9fd 100644 --- a/benchmark/test_res2net.py +++ b/benchmark/test_res2net.py @@ -3,6 +3,10 @@ from flowvision.models.res2net import res2net50_26w_4s +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_res2net50_26w_4s_batch_size1( benchmark, net=res2net50_26w_4s, input_shape=[1, 3, 224, 224] @@ -11,6 +15,10 @@ def test_res2net50_26w_4s_batch_size1( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_res2net50_26w_4s_batch_size2( benchmark, net=res2net50_26w_4s, input_shape=[2, 3, 224, 224] @@ -19,6 +27,10 @@ def test_res2net50_26w_4s_batch_size2( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_res2net50_26w_4s_batch_size4( benchmark, net=res2net50_26w_4s, input_shape=[4, 3, 224, 224] @@ -27,6 +39,10 @@ def test_res2net50_26w_4s_batch_size4( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_res2net50_26w_4s_batch_size8( benchmark, net=res2net50_26w_4s, input_shape=[8, 3, 224, 224] @@ -35,6 +51,10 @@ def test_res2net50_26w_4s_batch_size8( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_res2net50_26w_4s_batch_size16( benchmark, net=res2net50_26w_4s, input_shape=[16, 3, 224, 224] diff --git a/benchmark/test_res_mlp.py b/benchmark/test_res_mlp.py index beb74d51..b2ef7de1 100644 --- a/benchmark/test_res_mlp.py +++ b/benchmark/test_res_mlp.py @@ -3,6 +3,10 @@ from flowvision.models.res_mlp import resmlp_12_224 +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_resmlp_12_224_batch_size1( benchmark, net=resmlp_12_224, input_shape=[1, 3, 224, 224] @@ -11,6 +15,10 @@ def test_resmlp_12_224_batch_size1( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_resmlp_12_224_batch_size2( benchmark, net=resmlp_12_224, input_shape=[2, 3, 224, 224] @@ -19,6 +27,10 @@ def test_resmlp_12_224_batch_size2( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_resmlp_12_224_batch_size4( benchmark, net=resmlp_12_224, input_shape=[4, 3, 224, 224] @@ -27,6 +39,10 @@ def test_resmlp_12_224_batch_size4( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_resmlp_12_224_batch_size8( benchmark, net=resmlp_12_224, input_shape=[8, 3, 224, 224] @@ -35,6 +51,10 @@ def test_resmlp_12_224_batch_size8( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_resmlp_12_224_batch_size16( benchmark, net=resmlp_12_224, input_shape=[16, 3, 224, 224] diff --git a/benchmark/test_rexnet.py b/benchmark/test_rexnet.py index e041ac8e..2c71a51c 100644 --- a/benchmark/test_rexnet.py +++ b/benchmark/test_rexnet.py @@ -4,6 +4,10 @@ from flowvision.models.rexnet_lite import rexnet_lite_1_0 +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_rexnetv1_1_0_batch_size1( benchmark, net=rexnetv1_1_0, input_shape=[1, 3, 224, 224] @@ -12,6 +16,10 @@ def test_rexnetv1_1_0_batch_size1( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_rexnetv1_1_0_batch_size2( benchmark, net=rexnetv1_1_0, input_shape=[2, 3, 224, 224] @@ -20,6 +28,10 @@ def test_rexnetv1_1_0_batch_size2( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_rexnetv1_1_0_batch_size4( benchmark, net=rexnetv1_1_0, input_shape=[4, 3, 224, 224] @@ -28,6 +40,10 @@ def test_rexnetv1_1_0_batch_size4( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_rexnetv1_1_0_batch_size8( benchmark, net=rexnetv1_1_0, input_shape=[8, 3, 224, 224] @@ -36,6 +52,10 @@ def test_rexnetv1_1_0_batch_size8( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_rexnetv1_1_0_batch_size16( benchmark, net=rexnetv1_1_0, input_shape=[16, 3, 224, 224] @@ -44,6 +64,10 @@ def test_rexnetv1_1_0_batch_size16( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_rexnet_lite_1_0_batch_size1( benchmark, net=rexnet_lite_1_0, input_shape=[1, 3, 224, 224] @@ -52,6 +76,10 @@ def test_rexnet_lite_1_0_batch_size1( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_rexnet_lite_1_0_batch_size2( benchmark, net=rexnet_lite_1_0, input_shape=[2, 3, 224, 224] @@ -60,6 +88,10 @@ def test_rexnet_lite_1_0_batch_size2( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_rexnet_lite_1_0_batch_size4( benchmark, net=rexnet_lite_1_0, input_shape=[4, 3, 224, 224] @@ -68,6 +100,10 @@ def test_rexnet_lite_1_0_batch_size4( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_rexnet_lite_1_0_batch_size8( benchmark, net=rexnet_lite_1_0, input_shape=[8, 3, 224, 224] @@ -76,6 +112,10 @@ def test_rexnet_lite_1_0_batch_size8( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_rexnet_lite_1_0_batch_size16( benchmark, net=rexnet_lite_1_0, input_shape=[16, 3, 224, 224] diff --git a/benchmark/test_squeezenet.py b/benchmark/test_squeezenet.py index 34a61ffd..07a95507 100644 --- a/benchmark/test_squeezenet.py +++ b/benchmark/test_squeezenet.py @@ -3,6 +3,10 @@ from flowvision.models.squeezenet import squeezenet1_0 +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_squeezenet1_0_batch_size1( benchmark, net=squeezenet1_0, input_shape=[1, 3, 224, 224] @@ -11,6 +15,10 @@ def test_squeezenet1_0_batch_size1( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_squeezenet1_0_batch_size2( benchmark, net=squeezenet1_0, input_shape=[2, 3, 224, 224] @@ -19,6 +27,10 @@ def test_squeezenet1_0_batch_size2( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_squeezenet1_0_batch_size4( benchmark, net=squeezenet1_0, input_shape=[4, 3, 224, 224] @@ -27,6 +39,10 @@ def test_squeezenet1_0_batch_size4( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_squeezenet1_0_batch_size8( benchmark, net=squeezenet1_0, input_shape=[8, 3, 224, 224] @@ -35,6 +51,10 @@ def test_squeezenet1_0_batch_size8( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_squeezenet1_0_batch_size16( benchmark, net=squeezenet1_0, input_shape=[16, 3, 224, 224] diff --git a/benchmark/test_swin_transformer.py b/benchmark/test_swin_transformer.py index 469a0346..57a3953b 100644 --- a/benchmark/test_swin_transformer.py +++ b/benchmark/test_swin_transformer.py @@ -5,6 +5,10 @@ from flowvision.models.swin_transformer import swin_base_patch4_window7_224 +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_swin_tiny_patch4_window7_224_batch_size1( benchmark, net=swin_tiny_patch4_window7_224, input_shape=[1, 3, 224, 224] @@ -13,6 +17,10 @@ def test_swin_tiny_patch4_window7_224_batch_size1( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_swin_tiny_patch4_window7_224_batch_size2( benchmark, net=swin_tiny_patch4_window7_224, input_shape=[2, 3, 224, 224] @@ -21,6 +29,10 @@ def test_swin_tiny_patch4_window7_224_batch_size2( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_swin_tiny_patch4_window7_224_batch_size4( benchmark, net=swin_tiny_patch4_window7_224, input_shape=[4, 3, 224, 224] @@ -29,6 +41,10 @@ def test_swin_tiny_patch4_window7_224_batch_size4( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_swin_tiny_patch4_window7_224_batch_size8( benchmark, net=swin_tiny_patch4_window7_224, input_shape=[8, 3, 224, 224] @@ -37,6 +53,10 @@ def test_swin_tiny_patch4_window7_224_batch_size8( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_swin_tiny_patch4_window7_224_batch_size16( benchmark, net=swin_tiny_patch4_window7_224, input_shape=[16, 3, 224, 224] @@ -45,6 +65,10 @@ def test_swin_tiny_patch4_window7_224_batch_size16( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_swin_small_patch4_window7_224_batch_size1( benchmark, net=swin_small_patch4_window7_224, input_shape=[1, 3, 224, 224] @@ -53,6 +77,10 @@ def test_swin_small_patch4_window7_224_batch_size1( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_swin_small_patch4_window7_224_batch_size2( benchmark, net=swin_small_patch4_window7_224, input_shape=[2, 3, 224, 224] @@ -61,6 +89,10 @@ def test_swin_small_patch4_window7_224_batch_size2( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_swin_small_patch4_window7_224_batch_size4( benchmark, net=swin_small_patch4_window7_224, input_shape=[4, 3, 224, 224] @@ -69,6 +101,10 @@ def test_swin_small_patch4_window7_224_batch_size4( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_swin_small_patch4_window7_224_batch_size8( benchmark, net=swin_small_patch4_window7_224, input_shape=[8, 3, 224, 224] @@ -77,6 +113,10 @@ def test_swin_small_patch4_window7_224_batch_size8( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_swin_small_patch4_window7_224_batch_size16( benchmark, net=swin_small_patch4_window7_224, input_shape=[16, 3, 224, 224] @@ -85,6 +125,10 @@ def test_swin_small_patch4_window7_224_batch_size16( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_swin_base_patch4_window7_224_batch_size1( benchmark, net=swin_base_patch4_window7_224, input_shape=[1, 3, 224, 224] diff --git a/benchmark/test_uniformer.py b/benchmark/test_uniformer.py index b360d4fc..5b38a15c 100644 --- a/benchmark/test_uniformer.py +++ b/benchmark/test_uniformer.py @@ -46,6 +46,10 @@ def test_uniformer_base_batch_size16( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_uniformer_base_ls_batch_size1( benchmark, net=uniformer_base_ls, input_shape=[1, 3, 224, 224] @@ -54,6 +58,10 @@ def test_uniformer_base_ls_batch_size1( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_uniformer_base_ls_batch_size2( benchmark, net=uniformer_base_ls, input_shape=[2, 3, 224, 224] @@ -62,6 +70,10 @@ def test_uniformer_base_ls_batch_size2( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_uniformer_base_ls_batch_size4( benchmark, net=uniformer_base_ls, input_shape=[4, 3, 224, 224] @@ -70,6 +82,10 @@ def test_uniformer_base_ls_batch_size4( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_uniformer_base_ls_batch_size8( benchmark, net=uniformer_base_ls, input_shape=[8, 3, 224, 224] @@ -78,6 +94,10 @@ def test_uniformer_base_ls_batch_size8( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_uniformer_base_ls_batch_size16( benchmark, net=uniformer_base_ls, input_shape=[16, 3, 224, 224] @@ -86,6 +106,10 @@ def test_uniformer_base_ls_batch_size16( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_uniformer_small_batch_size1( benchmark, net=uniformer_small, input_shape=[1, 3, 224, 224] @@ -94,6 +118,10 @@ def test_uniformer_small_batch_size1( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_uniformer_small_batch_size2( benchmark, net=uniformer_small, input_shape=[2, 3, 224, 224] @@ -102,6 +130,10 @@ def test_uniformer_small_batch_size2( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_uniformer_small_batch_size4( benchmark, net=uniformer_small, input_shape=[4, 3, 224, 224] @@ -110,6 +142,10 @@ def test_uniformer_small_batch_size4( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_uniformer_small_batch_size8( benchmark, net=uniformer_small, input_shape=[8, 3, 224, 224] @@ -118,6 +154,10 @@ def test_uniformer_small_batch_size8( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_uniformer_small_batch_size16( benchmark, net=uniformer_small, input_shape=[16, 3, 224, 224] @@ -126,6 +166,10 @@ def test_uniformer_small_batch_size16( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_uniformer_small_plus_batch_size1( benchmark, net=uniformer_small_plus, input_shape=[1, 3, 224, 224] @@ -134,6 +178,10 @@ def test_uniformer_small_plus_batch_size1( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_uniformer_small_plus_batch_size2( benchmark, net=uniformer_small_plus, input_shape=[2, 3, 224, 224] @@ -142,6 +190,10 @@ def test_uniformer_small_plus_batch_size2( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_uniformer_small_plus_batch_size4( benchmark, net=uniformer_small_plus, input_shape=[4, 3, 224, 224] @@ -150,6 +202,10 @@ def test_uniformer_small_plus_batch_size4( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_uniformer_small_plus_batch_size8( benchmark, net=uniformer_small_plus, input_shape=[8, 3, 224, 224] @@ -158,6 +214,10 @@ def test_uniformer_small_plus_batch_size8( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_uniformer_small_plus_batch_size16( benchmark, net=uniformer_small_plus, input_shape=[16, 3, 224, 224] diff --git a/benchmark/test_vision_transformer.py b/benchmark/test_vision_transformer.py index fef6cc70..695230b4 100644 --- a/benchmark/test_vision_transformer.py +++ b/benchmark/test_vision_transformer.py @@ -5,6 +5,10 @@ from flowvision.models.vision_transformer import vit_base_patch16_224 +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_vit_tiny_patch16_224_batch_size1( benchmark, net=vit_tiny_patch16_224, input_shape=[1, 3, 224, 224] @@ -13,6 +17,10 @@ def test_vit_tiny_patch16_224_batch_size1( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_vit_tiny_patch16_224_batch_size2( benchmark, net=vit_tiny_patch16_224, input_shape=[2, 3, 224, 224] @@ -21,6 +29,10 @@ def test_vit_tiny_patch16_224_batch_size2( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_vit_tiny_patch16_224_batch_size4( benchmark, net=vit_tiny_patch16_224, input_shape=[4, 3, 224, 224] @@ -29,6 +41,10 @@ def test_vit_tiny_patch16_224_batch_size4( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_vit_tiny_patch16_224_batch_size8( benchmark, net=vit_tiny_patch16_224, input_shape=[8, 3, 224, 224] @@ -37,6 +53,10 @@ def test_vit_tiny_patch16_224_batch_size8( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_vit_tiny_patch16_224_batch_size16( benchmark, net=vit_tiny_patch16_224, input_shape=[16, 3, 224, 224] @@ -45,6 +65,10 @@ def test_vit_tiny_patch16_224_batch_size16( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_vit_small_patch16_224_batch_size1( benchmark, net=vit_small_patch16_224, input_shape=[1, 3, 224, 224] @@ -53,6 +77,10 @@ def test_vit_small_patch16_224_batch_size1( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_vit_small_patch16_224_batch_size2( benchmark, net=vit_small_patch16_224, input_shape=[2, 3, 224, 224] @@ -61,6 +89,10 @@ def test_vit_small_patch16_224_batch_size2( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_vit_small_patch16_224_batch_size4( benchmark, net=vit_small_patch16_224, input_shape=[4, 3, 224, 224] @@ -69,6 +101,10 @@ def test_vit_small_patch16_224_batch_size4( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_vit_small_patch16_224_batch_size8( benchmark, net=vit_small_patch16_224, input_shape=[8, 3, 224, 224] @@ -77,6 +113,10 @@ def test_vit_small_patch16_224_batch_size8( benchmark(run, model, x, optimizer) +@unittest.skipUnless( + os.getenv("ONEFLOW_BENCHMARK_ALL") == "1", + "set ONEFLOW_BENCHMARK_ALL=1 to run this test", +) @oneflow_benchmark.ci_settings(compare={"median": "5%"}) def test_vit_small_patch16_224_batch_size16( benchmark, net=vit_small_patch16_224, input_shape=[16, 3, 224, 224]