add testing scripts

llm_deploy/vllm_test.py

@@ -1,6 +1,23 @@
-from vllm import LLM
+from vllm import LLM, SamplingParams
+from vllm.model_executor.adapters import lora
 
-prompts = ["Hello, my name is", "The capital of France is"]  # Sample prompts.
-llm = LLM(model="../llm_ft/vicuna-7b-1.5")  # Create an LLM.
-outputs = llm.generate(prompts)  # Generate texts from the prompts.
-print(outputs)
+# Create an LLM.
+llm = LLM(model="../llm_ft/vicuna-7b-1.5", gpu_memory_utilization=0.05)
+
+# Add LoRA adapter
+lora.LoRAModel.from_pretrained(llm.llm_engine.workers[0].model, "../llm_ft/checkpoints/checkpoint-1200")
+
+prompts = [
+    "Hello, my name is",
+    "The capital of France is",
+    "The future of AI is",
+]
+
+sampling_params = SamplingParams(temperature=0, top_k=-1)
+
+outputs = llm.generate(prompts, sampling_params)
+
+for output in outputs:
+    prompt = output.prompt
+    generated_text = output.outputs[0].text
+    print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")