Merge pull request #205 from wandb/task/upgrades-9-24

Update frontend deps, setup uv, add support for custom endpoint, README

README.md

@@ -18,16 +18,21 @@ OpenUI let's you describe UI using your imagination, then see it rendered live.
 
 ## Running Locally
 
-OpenUI supports [OpenAI](https://platform.openai.com/api-keys), [Groq](https://console.groq.com/keys), and any model [LiteLLM](https://docs.litellm.ai/docs/) supports such as [Gemini](https://aistudio.google.com/app/apikey) or [Anthropic (Claude)](https://console.anthropic.com/settings/keys).  The following environment variables are optional, but need to be set in your environment for these services to work:
+OpenUI supports [OpenAI](https://platform.openai.com/api-keys), [Groq](https://console.groq.com/keys), and any model [LiteLLM](https://docs.litellm.ai/docs/) supports such as [Gemini](https://aistudio.google.com/app/apikey) or [Anthropic (Claude)](https://console.anthropic.com/settings/keys).  The following environment variables are optional, but need to be set in your environment for alternative models to work:
 
 - **OpenAI** `OPENAI_API_KEY`
 - **Groq** `GROQ_API_KEY`
 - **Gemini** `GEMINI_API_KEY`
 - **Anthropic** `ANTHROPIC_API_KEY`
 - **Cohere** `COHERE_API_KEY`
 - **Mistral** `MISTRAL_API_KEY`
+- **OpenAI Compatible** `OPENAI_COMPATIBLE_ENDPOINT` and `OPENAI_COMPATIBLE_API_KEY`
 
-You can also use models available to [Ollama](https://ollama.com).  [Install Ollama](https://ollama.com/download) and pull a model like [Llava](https://ollama.com/library/llava).  If Ollama is not running on http://127.0.0.1:11434, you can set the `OLLAMA_HOST` environment variable to the host and port of your Ollama instance.
+For example, if you're running a tool like [localai](https://localai.io/) you can set `OPENAI_COMPATIBLE_ENDPOINT` and optionally `OPENAI_COMPATIBLE_API_KEY` to have the models available listed in the UI's model selector under LiteLLM.
+
+### Ollama
+
+You can also use models available to [Ollama](https://ollama.com).  [Install Ollama](https://ollama.com/download) and pull a model like [Llava](https://ollama.com/library/llava).  If Ollama is not running on http://127.0.0.1:11434, you can set the `OLLAMA_HOST` environment variable to the host and port of your Ollama instance.  For example when running in docker you'll need to point to http://host.docker.internal:11434 as shown below.
 
 ### Docker (preferred)
 
@@ -43,23 +48,15 @@ Now you can goto [http://localhost:7878](http://localhost:7878) and generate new
 
 ### From Source / Python
 
-Assuming you have git and python installed:
-
-> **Note:** There's a .python-version file that specifies **openui** as the virtual env name.  Assuming you have pyenv and pyenv-virtualenv you can run the following from the root of the repository or just run `pyenv local 3.X` where X is the version of python you have installed.
-> ```bash
-> pyenv virtualenv 3.12.2 openui
-> pyenv local openui
-> ```
+Assuming you have git and [uv](https://github.com/astral-sh/uv) installed:
 
 ```bash
 git clone https://github.com/wandb/openui
 cd openui/backend
-# You probably want to do this from a virtual environment
-pip install .
+uv sync --frozen --extra litellm
+source .venv/bin/activate
 # Set API keys for any LLM's you want to use
 export OPENAI_API_KEY=xxx
-# You may change the base url to use an OpenAI-compatible api by setting the OPENAI_BASE_URL environment variable
-# export OPENAI_BASE_URL=https://api.myopenai.com/v1
 python -m openui
 ```
 
@@ -82,6 +79,7 @@ To use litellm from source you can run:
 ```bash
 pip install .[litellm]
 export ANTHROPIC_API_KEY=xxx
+export OPENAI_COMPATIBLE_ENDPOINT=http://localhost:8080/v1
 python -m openui --litellm
 ```
 

backend/.python-version

@@ -1 +1 @@
-openui
+3.12

backend/Dockerfile

@@ -1,25 +1,27 @@
 # Build the virtualenv as a separate step: Only re-execute this step when pyproject.toml changes
-FROM python:3.12-bookworm AS build-venv
+FROM ghcr.io/astral-sh/uv:python3.12-bookworm-slim AS builder
 
-WORKDIR /build
-COPY pyproject.toml .
-COPY README.md .
-
-RUN mkdir -p openui/util && \
-    python -m venv /venv && \
-    /venv/bin/pip install --upgrade pip setuptools wheel && \
-    /venv/bin/pip install --disable-pip-version-check .[litellm]
+WORKDIR /app
 
-# Copy the virtualenv into a distroless image
-FROM python:3.12-slim-bookworm
+ENV UV_LINK_MODE=copy UV_COMPILE_BYTECODE=1
 
-ENV PATH="/venv/bin:$PATH"
+RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=bind,source=uv.lock,target=uv.lock \
+    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
+    uv sync --frozen --extra litellm --no-install-project --no-dev
 
-COPY --from=build-venv /venv /venv
 COPY . /app
 
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync --frozen --extra litellm --no-dev
+
+# Copy the virtualenv into a distroless image
+FROM ghcr.io/astral-sh/uv:python3.12-bookworm-slim
+
 WORKDIR /app
 
-RUN pip install --no-deps -U /app
+COPY --from=builder --chown=app:app /app /app
+
+ENV PATH="/app/.venv/bin:$PATH"
 
 ENTRYPOINT ["python", "-m", "openui", "--litellm"]

backend/openui/config.py

@@ -39,6 +39,9 @@ class Env(Enum):
         SESSION_KEY = secrets.token_hex(32)
         with env_path.open("w") as f:
             f.write(f"OPENUI_SESSION_KEY={SESSION_KEY}")
+# Set the LITELLM_MASTER_KEY to a random value if it's not already set
+if os.getenv("LITELLM_MASTER_KEY") is None:
+    os.environ["LITELLM_MASTER_KEY"] = "sk-{SESSION_KEY}"
 # GPT 3.5 is 0.0005 per 1k tokens input and 0.0015 output
 # 700k puts us at a max of $1.00 spent per user over a 48 hour period
 MAX_TOKENS = int(os.getenv("OPENUI_MAX_TOKENS", "700000"))
@@ -61,5 +64,5 @@ class Env(Enum):
 OPENAI_API_KEY = os.getenv("OPENAI_API_KEY", "xxx")
 GROQ_BASE_URL = os.getenv("GROQ_BASE_URL", "https://api.groq.com/openai/v1")
 GROQ_API_KEY = os.getenv("GROQ_API_KEY")
-LITELLM_API_KEY = os.getenv("LITELLM_API_KEY", "xxx")
+LITELLM_API_KEY = os.getenv("LITELLM_API_KEY", os.getenv("LITELLM_MASTER_KEY"))
 LITELLM_BASE_URL = os.getenv("LITELLM_BASE_URL", "http://0.0.0.0:4000")

backend/openui/litellm.py

@@ -1,6 +1,8 @@
 import yaml
 import os
 import tempfile
+import openai
+from .logs import logger
 
 
 def generate_config():
@@ -95,6 +97,29 @@ def generate_config():
             ]
         )
 
+    if "OPENAI_COMPATIBLE_ENDPOINT" in os.environ:
+        client = openai.OpenAI(
+            api_key=os.getenv("OPENAI_COMPATIBLE_API_KEY"),
+            base_url=os.getenv("OPENAI_COMPATIBLE_ENDPOINT"),
+        )
+        try:
+            for model in client.models.list().data:
+                models.append(
+                    {
+                        "model_name": model.id,
+                        "litellm_params": {
+                            "model": f"openai/{model.id}",
+                            "api_key": os.getenv("OPENAI_COMPATIBLE_API_KEY"),
+                            "base_url": os.getenv("OPENAI_COMPATIBLE_ENDPOINT"),
+                        },
+                    }
+                )
+        except Exception as e:
+            logger.exception(
+                f"Error listing models for {os.getenv('OPENAI_COMPATIBLE_ENDPOINT')}: %s",
+                e,
+            )
+
     yaml_structure = {"model_list": models}
     with tempfile.NamedTemporaryFile(
         delete=False, mode="w", suffix=".yaml"

backend/openui/server.py

@@ -415,7 +415,7 @@ async def get_groq_models():
             d for d in (await groq.models.list()).data if not d.id.startswith("whisper")
         ]
     except Exception:
-        logger.warning("Couldn't connect to Ollama at %s", config.GROQ_BASE_URL)
+        logger.warning("Couldn't connect to Groq at %s", config.GROQ_BASE_URL)
         return []
 
 
@@ -594,13 +594,6 @@ def check_wandb_auth():
 
 wandb_enabled = check_wandb_auth()
 
-if not wandb_enabled:
-    try:
-        from weave.integrations.openai.openai_sdk import openai_patcher
-        openai_patcher.undo_patch()
-    except Exception:
-        pass
-
 class Server(uvicorn.Server):
     # TODO: this still isn't working for some reason, can't ctrl-c when not in dev mode
     def install_signal_handlers(self):