Update to latest vLLM upstream and Support vLLM on CPU

models/continuous_batching/cpu/meta-llama--Llama-2-7b-chat-hf.yaml

@@ -0,0 +1,41 @@
+deployment_config:
+ autoscaling_config:
+ min_replicas: 1
+ initial_replicas: 1
+ max_replicas: 8
+ target_num_ongoing_requests_per_replica: 24
+ metrics_interval_s: 10.0
+ look_back_period_s: 30.0
+ smoothing_factor: 0.6
+ downscale_delay_s: 300.0
+ upscale_delay_s: 15.0
+ max_concurrent_queries: 64
+ ray_actor_options: {}
+engine_config:
+ model_id: meta-llama/Llama-2-7b-chat-hf
+ hf_model_id: meta-llama/Llama-2-7b-chat-hf
+ type: VLLMEngine
+ engine_kwargs:
+ device: "cpu"
+ dtype: "float32"
+ trust_remote_code: true
+ max_num_batched_tokens: 4096
+ max_num_seqs: 64
+ gpu_memory_utilization: 0.95
+ max_total_tokens: 4096
+ generation:
+ prompt_format:
+ system: "<<SYS>>\n{instruction}\n<</SYS>>\n\n"
+ assistant: " {instruction} </s><s>"
+ trailing_assistant: ""
+ user: "[INST] {system}{instruction} [/INST]"
+ system_in_user: true
+ default_system_message: ""
+ stopping_sequences: []
+scaling_config:
+ num_workers: 1
+ num_gpus_per_worker: 0
+ num_cpus_per_worker: 32
+ placement_strategy: "STRICT_PACK"
+ resources_per_worker: {}
+

models/continuous_batching/meta-llama--Llama-2-7b-chat-hf.yaml

@@ -10,9 +10,7 @@ deployment_config:
  downscale_delay_s: 300.0
  upscale_delay_s: 15.0
  max_concurrent_queries: 64
- ray_actor_options:
- resources:
- accelerator_type_a10: 0.01
+ ray_actor_options: {}
 engine_config:
  model_id: meta-llama/Llama-2-7b-chat-hf
  hf_model_id: meta-llama/Llama-2-7b-chat-hf
@@ -34,8 +32,8 @@ engine_config:
  stopping_sequences: []
 scaling_config:
  num_workers: 1
- num_gpus_per_worker: 1
+ num_gpus_per_worker: 0
  num_cpus_per_worker: 8
  placement_strategy: "STRICT_PACK"
- resources_per_worker:
- accelerator_type_a10: 0.01
+ resources_per_worker: {}
+

rayllm/backend/llm/embedding/embedding_models.py

@@ -11,6 +11,7 @@
  ModelType,
  S3MirrorConfig,
 )
+from pydantic import ConfigDict
 
 logger = logging.getLogger(__name__)
 
@@ -20,8 +21,7 @@ class EmbeddingOptimize(str, Enum):
 
 
 class EmbeddingEngineConfig(EngineConfig):
- class Config:
- use_enum_values = True
+ model_config = ConfigDict(use_enum_values=True)
 
  type: EngineType = EngineType.EmbeddingEngine
  model_type: ModelType = ModelType.embedding

rayllm/backend/llm/trtllm/trtllm_models.py

@@ -10,6 +10,7 @@
  S3MirrorConfig,
  SamplingParams,
 )
+from pydantic import ConfigDict
 
 try:
  from tensorrt_llm.libs import trt_llm_engine_py as trt_py
@@ -23,9 +24,7 @@ class TRTLLMGPTServeConfig(BaseModelExtended):
  max_tokens_in_paged_kv_cache: int = None
  kv_cache_free_gpu_mem_fraction: float = None
  enable_trt_overlap: bool = None
-
- class Config:
- arbitrary_types_allowed = True
+ model_config = ConfigDict(arbitrary_types_allowed=True)
 
  @classmethod
  def from_engine_config(

rayllm/backend/llm/vllm/vllm_compatibility.py

@@ -1,14 +1,10 @@
 import logging
 import time
-from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Type, Union
+from typing import TYPE_CHECKING, Any, List, Optional, Type, Union
 
 import ray
 from ray.util.placement_group import PlacementGroup
 from transformers.dynamic_module_utils import init_hf_modules
-from vllm.config import CacheConfig as VllmCacheConfig
-from vllm.config import ModelConfig as VllmModelConfig
-from vllm.config import ParallelConfig as VllmParallelConfig
-from vllm.config import SchedulerConfig as VllmSchedulerConfig
 from vllm.engine.arg_utils import AsyncEngineArgs
 from vllm.engine.async_llm_engine import AsyncLLMEngine, AsyncStream, _AsyncLLMEngine
 
@@ -32,13 +28,8 @@
 
 logger = logging.getLogger(__name__)
 
-VllmConfigs = Tuple[
- VllmCacheConfig, VllmModelConfig, VllmParallelConfig, VllmSchedulerConfig
-]
-
-
 class AviaryLLMEngine(_AsyncLLMEngine):
- def __init__(self, *args, runtime_env: dict, **kwargs):
+ def __init__(self, *args, runtime_env: dict = {}, **kwargs):
  self.runtime_env = runtime_env
  super().__init__(*args, **kwargs)
 
@@ -106,15 +97,17 @@ def _record_system_stats(self, *args, **kwargs):
  return last_stats
 
 
-def _get_vllm_engine_config(vllm_app) -> Tuple[AsyncEngineArgs, VllmConfigs]:
- # Generate engine arguements and engine configs
+def _get_vllm_engine_config(vllm_app) -> AsyncEngineArgs:
+ device = vllm_app.engine_config.engine_kwargs.get("device", "gpu")
 
+ # Generate engine arguements and engine configs
  async_engine_args = AsyncEngineArgs(
  # This is the local path on disk, or the hf model id
  # If it is the hf_model_id, vllm automatically downloads the correct model.
  **dict(
  model=vllm_app.engine_config.actual_hf_model_id,
- worker_use_ray=True,
+ # vLLM for CPU doesn't support Ray workers for model parallelism yet
+ worker_use_ray=True if device == "gpu" else False,
  engine_use_ray=False,
  tensor_parallel_size=vllm_app.placement_config.world_size,
  max_model_len=vllm_app.engine_config.max_total_tokens,
@@ -123,8 +116,8 @@ def _get_vllm_engine_config(vllm_app) -> Tuple[AsyncEngineArgs, VllmConfigs]:
  **vllm_app.engine_config.get_initialization_kwargs(),
  )
  )
- configs = async_engine_args.create_engine_configs()
- return async_engine_args, configs
+
+ return async_engine_args
 
 
 class AviaryAsyncLLMEngine(AsyncLLMEngine):
@@ -167,23 +160,12 @@ def from_llm_app(
  # torch to have access to CUDA devices. We use a remote task
  # with `num_gpus` set here, so the type check happens in an environment
  # with `CUDA_VISIBLE_DEVICES` set.
- engine_args, engine_configs = ray.get(
+ engine_args = ray.get(
  ray.remote(_get_vllm_engine_config)
  .options(**scaling_config)
  .remote(vllm_app)
  )
 
- # Create the async LLM engine.
- engine = cls(
- engine_args.worker_use_ray,
- engine_args.engine_use_ray,
- *engine_configs,
- None,
- placement_group,
- runtime_env=runtime_env,
- log_requests=not engine_args.disable_log_requests,
- log_stats=not engine_args.disable_log_stats,
- max_log_len=engine_args.max_log_len,
- start_engine_loop=True,
- )
+ engine = cls.from_engine_args(engine_args, start_engine_loop=True)
+
  return engine

rayllm/backend/llm/vllm/vllm_engine.py

@@ -59,7 +59,9 @@ def __init__(
  self.llm_app = llm_app.copy(deep=True)
  self.engine_config = llm_app.engine_config
  self.placement_config = llm_app.placement_config
- if not (self.placement_config.scaling_config.num_gpus_per_worker > 0):
+
+ device = self.engine_config.engine_kwargs.get("device", "gpu")
+ if device == "gpu" and not (self.placement_config.scaling_config.num_gpus_per_worker > 0):
  raise ValueError("The VLLM Engine Requires > 0 GPUs to run.")
 
  self.node_initializer = node_initializer or LLMNodeInitializer(

rayllm/backend/server/config_models.py

@@ -0,0 +1,86 @@
+from typing import Optional
+from pydantic import (
+ BaseModel,
+ Field,
+ NonNegativeFloat,
+ NonNegativeInt,
+ PositiveFloat,
+ PositiveInt,
+ field_validator,
+)
+
+# Adapted from ray.serve.config.AutoscalingConfig
+# Port it here as the original AutoscalingConfig model is pydantic V1
+class AutoscalingConfig(BaseModel):
+ """Config for the Serve Autoscaler."""
+
+ # Please keep these options in sync with those in
+ # `src/ray/protobuf/serve.proto`.
+
+ # Publicly exposed options
+ min_replicas: NonNegativeInt = 1
+ initial_replicas: Optional[NonNegativeInt] = None
+ max_replicas: PositiveInt = 1
+
+ # DEPRECATED: replaced by target_ongoing_requests
+ target_num_ongoing_requests_per_replica: PositiveFloat = Field(
+ default=1.0,
+ description="[DEPRECATED] Please use `target_ongoing_requests` instead.",
+ )
+ # Will default to 1.0 in the future.
+ target_ongoing_requests: Optional[PositiveFloat] = None
+
+ # How often to scrape for metrics
+ metrics_interval_s: PositiveFloat = 10.0
+ # Time window to average over for metrics.
+ look_back_period_s: PositiveFloat = 30.0
+
+ # DEPRECATED
+ smoothing_factor: PositiveFloat = 1.0
+ # DEPRECATED: replaced by `downscaling_factor`
+ upscale_smoothing_factor: Optional[PositiveFloat] = Field(
+ default=None, description="[DEPRECATED] Please use `upscaling_factor` instead."
+ )
+ # DEPRECATED: replaced by `upscaling_factor`
+ downscale_smoothing_factor: Optional[PositiveFloat] = Field(
+ default=None,
+ description="[DEPRECATED] Please use `downscaling_factor` instead.",
+ )
+
+ # Multiplicative "gain" factor to limit scaling decisions
+ upscaling_factor: Optional[PositiveFloat] = None
+ downscaling_factor: Optional[PositiveFloat] = None
+
+ # How frequently to make autoscaling decisions
+ # loop_period_s: float = CONTROL_LOOP_PERIOD_S
+ # How long to wait before scaling down replicas
+ downscale_delay_s: NonNegativeFloat = 600.0
+ # How long to wait before scaling up replicas
+ upscale_delay_s: NonNegativeFloat = 30.0
+
+ @field_validator("max_replicas")
+ def replicas_settings_valid(cls, max_replicas, values):
+ min_replicas = values.data.get("min_replicas")
+ initial_replicas = values.data.get("initial_replicas")
+ if min_replicas is not None and max_replicas < min_replicas:
+ raise ValueError(
+ f"max_replicas ({max_replicas}) must be greater than "
+ f"or equal to min_replicas ({min_replicas})!"
+ )
+
+ if initial_replicas is not None:
+ if initial_replicas < min_replicas:
+ raise ValueError(
+ f"min_replicas ({min_replicas}) must be less than "
+ f"or equal to initial_replicas ({initial_replicas})!"
+ )
+ elif initial_replicas > max_replicas:
+ raise ValueError(
+ f"max_replicas ({max_replicas}) must be greater than "
+ f"or equal to initial_replicas ({initial_replicas})!"
+ )
+
+ return max_replicas
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)