From 66a0eb46731d2c2c5ea2c035d4b83fc69fa0bd92 Mon Sep 17 00:00:00 2001 From: Ross Wightman Date: Fri, 7 Jun 2024 16:09:25 -0700 Subject: [PATCH 1/2] Experimenting with tiny test models, how small can they go and be useful for regression tests? --- timm/models/byobnet.py | 24 +++++++++++++++++++++ timm/models/efficientnet.py | 36 +++++++++++++++++++++++++++++++ timm/models/vision_transformer.py | 18 +++++++++++++--- 3 files changed, 75 insertions(+), 3 deletions(-) diff --git a/timm/models/byobnet.py b/timm/models/byobnet.py index 02e258361..d05735d1b 100644 --- a/timm/models/byobnet.py +++ b/timm/models/byobnet.py @@ -1834,6 +1834,18 @@ def _init_weights(module, name='', zero_init_last=False): stem_type='one', stem_chs=64, ), + + test_tiny_resnet=ByoModelCfg( + blocks=( + ByoBlockCfg(type='basic', d=1, c=24, s=1, gs=1, br=0.25), + ByoBlockCfg(type='basic', d=1, c=32, s=2, gs=1, br=0.25), + ByoBlockCfg(type='basic', d=1, c=64, s=2, gs=1, br=0.25), + ByoBlockCfg(type='basic', d=1, c=128, s=2, gs=1, br=0.25), + ), + stem_chs=32, + stem_pool='maxpool', + act_layer='relu', + ), ) @@ -2035,6 +2047,11 @@ def _cfgr(url='', **kwargs): crop_pct=0.9, first_conv=('stem.conv_kxk.0.conv', 'stem.conv_scale.conv'), ), + + 'test_tiny_byobnet.untrained': _cfgr( + # hf_hub_id='timm/', + input_size=(3, 160, 160), crop_pct=0.875, pool_size=(5, 5), + ), }) @@ -2337,3 +2354,10 @@ def mobileone_s4(pretrained=False, **kwargs) -> ByobNet: """ """ return _create_byobnet('mobileone_s4', pretrained=pretrained, **kwargs) + + +@register_model +def test_tiny_byobnet(pretrained=False, **kwargs) -> ByobNet: + """ Minimal test ResNet (BYOB based) model. + """ + return _create_byobnet('test_tiny_byobnet', pretrained=pretrained, **kwargs) diff --git a/timm/models/efficientnet.py b/timm/models/efficientnet.py index 36059577e..57efb9078 100644 --- a/timm/models/efficientnet.py +++ b/timm/models/efficientnet.py @@ -1056,6 +1056,31 @@ def _gen_tinynet( return model +def _gen_test_efficientnet( + variant, channel_multiplier=1.0, depth_multiplier=1.0, pretrained=False, **kwargs): + """ Minimal test EfficientNet generator. + """ + arch_def = [ + ['cn_r1_k3_s1_e1_c16_skip'], + ['er_r1_k3_s2_e4_c24'], + ['er_r1_k3_s2_e4_c32'], + ['ir_r1_k3_s2_e4_c48_se0.25'], + ['ir_r1_k3_s2_e4_c64_se0.25'], + ] + round_chs_fn = partial(round_channels, multiplier=channel_multiplier, round_limit=0.) + model_kwargs = dict( + block_args=decode_arch_def(arch_def, depth_multiplier), + num_features=round_chs_fn(256), + stem_size=24, + round_chs_fn=round_chs_fn, + norm_layer=kwargs.pop('norm_layer', None) or partial(nn.BatchNorm2d, **resolve_bn_args(kwargs)), + act_layer=resolve_act_layer(kwargs, 'silu'), + **kwargs, + ) + model = _create_effnet(variant, pretrained, **model_kwargs) + return model + + def _cfg(url='', **kwargs): return { 'url': url, 'num_classes': 1000, 'input_size': (3, 224, 224), 'pool_size': (7, 7), @@ -1584,6 +1609,10 @@ def _cfg(url='', **kwargs): input_size=(3, 106, 106), pool_size=(4, 4), # int(224 * 0.475) url='https://github.com/huawei-noah/CV-Backbones/releases/download/v1.2.0/tinynet_e.pth', hf_hub_id='timm/'), + + "test_tiny_efficientnet.untrained": _cfg( + # hf_hub_id='timm/' + input_size=(3, 160, 160), pool_size=(5, 5)), }) @@ -2510,6 +2539,13 @@ def tinynet_e(pretrained=False, **kwargs) -> EfficientNet: return model +@register_model +def test_tiny_efficientnet(pretrained=False, **kwargs) -> EfficientNet: + model = _gen_test_efficientnet('test_tiny_efficientnet', pretrained=pretrained, **kwargs) + return model + + + register_model_deprecations(__name__, { 'tf_efficientnet_b0_ap': 'tf_efficientnet_b0.ap_in1k', 'tf_efficientnet_b1_ap': 'tf_efficientnet_b1.ap_in1k', diff --git a/timm/models/vision_transformer.py b/timm/models/vision_transformer.py index a3ca0990d..a8bc450f3 100644 --- a/timm/models/vision_transformer.py +++ b/timm/models/vision_transformer.py @@ -1929,13 +1929,16 @@ def _cfg(url: str = '', **kwargs) -> Dict[str, Any]: hf_hub_id='timm/', num_classes=11821, input_size=(3, 256, 256), crop_pct=0.95), - 'vit_base_patch16_reg4_gap_256': _cfg( + 'vit_base_patch16_reg4_gap_256.untrained': _cfg( input_size=(3, 256, 256)), - 'vit_so150m_patch16_reg4_gap_256': _cfg( + 'vit_so150m_patch16_reg4_gap_256.untrained': _cfg( input_size=(3, 256, 256)), - 'vit_so150m_patch16_reg4_map_256': _cfg( + 'vit_so150m_patch16_reg4_map_256.untrained': _cfg( input_size=(3, 256, 256)), + + 'test_tiny_vit.untrained': _cfg( + input_size=(3, 160, 160), crop_pct=0.875), } _quick_gelu_cfgs = [ @@ -3106,6 +3109,15 @@ def vit_so150m_patch16_reg4_gap_256(pretrained: bool = False, **kwargs) -> Visio return model +@register_model +def test_tiny_vit(pretrained: bool = False, **kwargs) -> VisionTransformer: + """ ViT-TestTiny + """ + model_args = dict(patch_size=16, embed_dim=64, depth=4, num_heads=1, mlp_ratio=3) + model = _create_vision_transformer('test_tiny_vit', pretrained=pretrained, **dict(model_args, **kwargs)) + return model + + register_model_deprecations(__name__, { 'vit_tiny_patch16_224_in21k': 'vit_tiny_patch16_224.augreg_in21k', 'vit_small_patch32_224_in21k': 'vit_small_patch32_224.augreg_in21k', From 55101028bb93a5cdeb89085b8ffbc9634160602b Mon Sep 17 00:00:00 2001 From: Ross Wightman Date: Tue, 9 Jul 2024 11:53:11 -0700 Subject: [PATCH 2/2] Rename test_tiny* -> test*. Fix ByobNet BasicBlock attn location and add test_byobnet model. --- timm/models/byobnet.py | 27 +++++++++++++++------------ timm/models/efficientnet.py | 6 +++--- timm/models/vision_transformer.py | 10 +++++----- 3 files changed, 23 insertions(+), 20 deletions(-) diff --git a/timm/models/byobnet.py b/timm/models/byobnet.py index d05735d1b..d4dfd6c9e 100644 --- a/timm/models/byobnet.py +++ b/timm/models/byobnet.py @@ -261,8 +261,9 @@ def init_weights(self, zero_init_last: bool = False): def forward(self, x): shortcut = x x = self.conv1_kxk(x) - x = self.conv2_kxk(x) x = self.attn(x) + x = self.conv2_kxk(x) + x = self.attn_last(x) x = self.drop_path(x) if self.shortcut is not None: x = x + self.shortcut(shortcut) @@ -439,7 +440,6 @@ def __init__( downsample, in_chs, out_chs, stride=stride, dilation=dilation, apply_act=False, layers=layers, ) - self.conv1_kxk = layers.conv_norm_act( in_chs, mid_chs, kernel_size, stride=stride, dilation=dilation[0], groups=groups, drop_layer=drop_block, @@ -1835,16 +1835,19 @@ def _init_weights(module, name='', zero_init_last=False): stem_chs=64, ), - test_tiny_resnet=ByoModelCfg( + test_byobnet=ByoModelCfg( blocks=( - ByoBlockCfg(type='basic', d=1, c=24, s=1, gs=1, br=0.25), - ByoBlockCfg(type='basic', d=1, c=32, s=2, gs=1, br=0.25), - ByoBlockCfg(type='basic', d=1, c=64, s=2, gs=1, br=0.25), - ByoBlockCfg(type='basic', d=1, c=128, s=2, gs=1, br=0.25), + ByoBlockCfg(type='edge', d=1, c=32, s=2, gs=0, br=0.5), + ByoBlockCfg(type='dark', d=1, c=64, s=2, gs=0, br=0.5), + ByoBlockCfg(type='basic', d=1, c=128, s=2, gs=32, br=0.25), + ByoBlockCfg(type='bottle', d=1, c=256, s=2, gs=64, br=0.25), ), - stem_chs=32, - stem_pool='maxpool', + stem_chs=24, + downsample='avg', + stem_pool='', act_layer='relu', + attn_layer='se', + attn_kwargs=dict(rd_ratio=0.25), ), ) @@ -2048,7 +2051,7 @@ def _cfgr(url='', **kwargs): first_conv=('stem.conv_kxk.0.conv', 'stem.conv_scale.conv'), ), - 'test_tiny_byobnet.untrained': _cfgr( + 'test_byobnet.untrained': _cfgr( # hf_hub_id='timm/', input_size=(3, 160, 160), crop_pct=0.875, pool_size=(5, 5), ), @@ -2357,7 +2360,7 @@ def mobileone_s4(pretrained=False, **kwargs) -> ByobNet: @register_model -def test_tiny_byobnet(pretrained=False, **kwargs) -> ByobNet: +def test_byobnet(pretrained=False, **kwargs) -> ByobNet: """ Minimal test ResNet (BYOB based) model. """ - return _create_byobnet('test_tiny_byobnet', pretrained=pretrained, **kwargs) + return _create_byobnet('test_byobnet', pretrained=pretrained, **kwargs) diff --git a/timm/models/efficientnet.py b/timm/models/efficientnet.py index 57efb9078..fe397b056 100644 --- a/timm/models/efficientnet.py +++ b/timm/models/efficientnet.py @@ -1610,7 +1610,7 @@ def _cfg(url='', **kwargs): url='https://github.com/huawei-noah/CV-Backbones/releases/download/v1.2.0/tinynet_e.pth', hf_hub_id='timm/'), - "test_tiny_efficientnet.untrained": _cfg( + "test_efficientnet.untrained": _cfg( # hf_hub_id='timm/' input_size=(3, 160, 160), pool_size=(5, 5)), }) @@ -2540,8 +2540,8 @@ def tinynet_e(pretrained=False, **kwargs) -> EfficientNet: @register_model -def test_tiny_efficientnet(pretrained=False, **kwargs) -> EfficientNet: - model = _gen_test_efficientnet('test_tiny_efficientnet', pretrained=pretrained, **kwargs) +def test_efficientnet(pretrained=False, **kwargs) -> EfficientNet: + model = _gen_test_efficientnet('test_efficientnet', pretrained=pretrained, **kwargs) return model diff --git a/timm/models/vision_transformer.py b/timm/models/vision_transformer.py index a8bc450f3..bcb7059e2 100644 --- a/timm/models/vision_transformer.py +++ b/timm/models/vision_transformer.py @@ -1937,7 +1937,7 @@ def _cfg(url: str = '', **kwargs) -> Dict[str, Any]: 'vit_so150m_patch16_reg4_map_256.untrained': _cfg( input_size=(3, 256, 256)), - 'test_tiny_vit.untrained': _cfg( + 'test_vit.untrained': _cfg( input_size=(3, 160, 160), crop_pct=0.875), } @@ -3110,11 +3110,11 @@ def vit_so150m_patch16_reg4_gap_256(pretrained: bool = False, **kwargs) -> Visio @register_model -def test_tiny_vit(pretrained: bool = False, **kwargs) -> VisionTransformer: - """ ViT-TestTiny +def test_vit(pretrained: bool = False, **kwargs) -> VisionTransformer: + """ ViT Test """ - model_args = dict(patch_size=16, embed_dim=64, depth=4, num_heads=1, mlp_ratio=3) - model = _create_vision_transformer('test_tiny_vit', pretrained=pretrained, **dict(model_args, **kwargs)) + model_args = dict(patch_size=16, embed_dim=64, depth=6, num_heads=2, mlp_ratio=3) + model = _create_vision_transformer('test_vit', pretrained=pretrained, **dict(model_args, **kwargs)) return model