From e4d288b45b89cee462a21ab264405f3f368adc21 Mon Sep 17 00:00:00 2001 From: Qiang Zhang Date: Thu, 25 Apr 2024 16:18:01 -0700 Subject: [PATCH] set bias=True for linear layer (#527) Summary: Pull Request resolved: https://github.com/facebookresearch/multimodal/pull/527 Pull Request resolved: https://github.com/facebookresearch/multimodal/pull/526 1. for output projection in text decoder, change bias=False to True. In many other places, e.g., LP head, ember's output module and LLAVA, they are using bias=True (which is default value in Linear). 2. add configuration of using MLP instead of attention pooler for vision adapter; Reviewed By: Bellaktris Differential Revision: D55897450 Privacy Context Container: 303860477774201 fbshipit-source-id: 8e012b0c3d37566364f216dbfa8aec389142afe1 --- tests/test_utils.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 856e2d67..a3462d78 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -192,8 +192,18 @@ def assert_expected_namedtuple( def init_weights_with_constant(model: nn.Module, constant: float = 1.0) -> None: - for p in model.parameters(): + for n, p in model.named_parameters(): nn.init.constant_(p, constant) + # reduce the change to the tests + for k in { + "text_projection.bias", + "pooled_projection.bias", + "output_projection.bias", + "vision_proj.bias", + }: + if n.endswith(k): + nn.init.constant_(p, 0.0) + break def tensor_hash(x: torch.tensor, scaling=0.05, buckets=1000) -> torch.tensor: