diff --git a/src/diffusers/loaders/lora.py b/src/diffusers/loaders/lora.py
index 8a5293947027..50454a70469e 100644
--- a/src/diffusers/loaders/lora.py
+++ b/src/diffusers/loaders/lora.py
@@ -395,8 +395,7 @@ def load_lora_into_unet(cls, state_dict, network_alphas, unet, adapter_name=None
         # their prefixes.
         keys = list(state_dict.keys())
         only_text_encoder = all(key.startswith(cls.text_encoder_name) for key in keys)
-
-        if any(key.startswith(cls.unet_name) for key in keys) and not only_text_encoder:
+        if not only_text_encoder:
             # Load the layers corresponding to UNet.
             logger.info(f"Loading {cls.unet_name}.")
             unet.load_attn_procs(