From 92bd0946ccc64c1b3cb6eb2f3d7a1e3eae9f092c Mon Sep 17 00:00:00 2001 From: Qiang Zhang Date: Tue, 9 Apr 2024 16:01:20 -0700 Subject: [PATCH] set bias=True for linear layer (#527) Summary: 1. for output projection in text decoder, change bias=False to True. In many other places, e.g., LP head, ember's output module and LLAVA, they are using bias=True (which is default value in Linear). 2. add configuration of using MLP instead of attention pooler for vision adapter; Differential Revision: D55897450 Privacy Context Container: 303860477774201 --- tests/test_utils.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 856e2d67..a3462d78 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -192,8 +192,18 @@ def assert_expected_namedtuple( def init_weights_with_constant(model: nn.Module, constant: float = 1.0) -> None: - for p in model.parameters(): + for n, p in model.named_parameters(): nn.init.constant_(p, constant) + # reduce the change to the tests + for k in { + "text_projection.bias", + "pooled_projection.bias", + "output_projection.bias", + "vision_proj.bias", + }: + if n.endswith(k): + nn.init.constant_(p, 0.0) + break def tensor_hash(x: torch.tensor, scaling=0.05, buckets=1000) -> torch.tensor: