diff --git a/docs/assets/nos_catalog_t4_complete.png b/docs/assets/nos_catalog_t4_complete.png new file mode 100644 index 00000000..515a9cd6 Binary files /dev/null and b/docs/assets/nos_catalog_t4_complete.png differ diff --git a/docs/cli/profile.md b/docs/cli/profile.md new file mode 100644 index 00000000..5939a791 --- /dev/null +++ b/docs/cli/profile.md @@ -0,0 +1,42 @@ +## ⏱️ Profiling Models + +`nos profile` supports model benchmarking across a number of axes including iterations per second, +memory footprint and utilization on CPU/GPU. +Currently, the nos profiler itself runs natively in the execution environment (i.e. outside +the NOS server), so you'll need to install both the `server` and `test` dependencies alongside +your existing NOS installation. + +```bash +pip install torch-nos[server,test] +``` + +## NOS Profile Commands + +### all +Profile all models + +If you have the time, you can construct a profiling catalog on your machine in its entirety with: +```bash +nos profile all +``` +### rebuild-catalog +Generate a fresh catalog with +```bash +nos profile rebuild-catalog +``` +### model +You can also profile specific models with +```bash +nos profile model -m openai/clip-vit-large-patch14 +``` +### method +Or an entire method type with +```bash +nos profile method -m encode_image +``` +to benchmark e.g. all image embedding models. + +### list +Dump the nos profiling catalog with `nos profile list` + +![NOS Profile List](../assets/nos_catalog_t4_complete.png) \ No newline at end of file diff --git a/nos/common/spec.py b/nos/common/spec.py index 400ac9c2..ebb0761b 100644 --- a/nos/common/spec.py +++ b/nos/common/spec.py @@ -433,19 +433,30 @@ def load(self, model_method_id: Any) -> "ModelSpec": def load_profile_catalog(self) -> "ModelSpecMetadataCatalog": """Load the model profiles from a JSON catalog.""" + import logging + import os + from pathlib import Path + import pandas as pd - if not NOS_PROFILE_CATALOG_PATH.exists(): - raise FileNotFoundError(f"Model metadata catalog not found, path={NOS_PROFILE_CATALOG_PATH}.") + logger = logging.getLogger(__name__) - import logging + catalog_path = NOS_PROFILE_CATALOG_PATH - logger = logging.getLogger(__name__) - debug_str = "Loading profiling catalog from " + str(NOS_PROFILE_CATALOG_PATH) + # Check if we have NOS_PROFILE_CATALOG_PATH_OVERRIDE in the environment + if os.environ.get("NOS_PROFILE_CATALOG_PATH_OVERRIDE"): + catalog_path = Path(os.environ["NOS_PROFILE_CATALOG_PATH_OVERRIDE"]) + logger.debug(f"Using custom profile catalog [path={catalog_path}]") + + if not catalog_path.exists(): + # Make sure the catalog (either default or custom) exists + raise FileNotFoundError(f"Model metadata catalog not found, path={catalog_path}.") + + debug_str = "Loading profiling catalog from " + str(catalog_path) logger.info(debug_str) # Read the catalog - df = pd.read_json(str(NOS_PROFILE_CATALOG_PATH), orient="records") + df = pd.read_json(str(catalog_path), orient="records") columns = df.columns # Check if the catalog is valid with the required columns