[LoRA] fix vanilla fine-tuned lora loading. (#8691)

fix vanilla fine-tuned lora loading.

src/diffusers/loaders/lora.py

@@ -395,8 +395,7 @@ def load_lora_into_unet(cls, state_dict, network_alphas, unet, adapter_name=None
         # their prefixes.
         keys = list(state_dict.keys())
         only_text_encoder = all(key.startswith(cls.text_encoder_name) for key in keys)
-
-        if any(key.startswith(cls.unet_name) for key in keys) and not only_text_encoder:
+        if not only_text_encoder:
             # Load the layers corresponding to UNet.
             logger.info(f"Loading {cls.unet_name}.")
             unet.load_attn_procs(