From 4ecf3e5450bdaa72c01550c969e7a982835d1a33 Mon Sep 17 00:00:00 2001 From: pelikhan Date: Wed, 11 Dec 2024 09:04:36 -0800 Subject: [PATCH] =?UTF-8?q?feat:=20=F0=9F=94=A7=20add=20LLMProviderFeature?= =?UTF-8?q?s=20component=20and=20update=20docs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/public/schemas/llms.json | 22 ++------ docs/src/components/LLMProviderFeatures.astro | 56 +++++++++++++++++++ .../docs/getting-started/configuration.mdx | 38 ++++++++++--- packages/core/src/llms.json | 13 +++-- 4 files changed, 100 insertions(+), 29 deletions(-) create mode 100644 docs/src/components/LLMProviderFeatures.astro diff --git a/docs/public/schemas/llms.json b/docs/public/schemas/llms.json index 82970443d3..20f3395696 100644 --- a/docs/public/schemas/llms.json +++ b/docs/public/schemas/llms.json @@ -13,24 +13,6 @@ "properties": { "id": { "type": "string", - "enum": [ - "openai", - "github", - "azure", - "azure_serverless", - "azure_serverless_models", - "anthropic", - "googe", - "huggingface", - "transformers", - "ollama", - "mistal", - "lmstudio", - "jan", - "alibaba", - "llamafile", - "litellm" - ], "description": "Identifier for the LLM provider" }, "detail": { @@ -60,6 +42,10 @@ "tools": { "type": "boolean", "description": "Indicates if tools are supported" + }, + "openaiCompatibility": { + "type": "string", + "description": "Uses OpenAI API compatibility layer documentation URL" } }, "additionalProperties": false, diff --git a/docs/src/components/LLMProviderFeatures.astro b/docs/src/components/LLMProviderFeatures.astro new file mode 100644 index 0000000000..014fd30c67 --- /dev/null +++ b/docs/src/components/LLMProviderFeatures.astro @@ -0,0 +1,56 @@ +--- +import LLMS from "../../../packages/core/src/llms.json" + +interface Props { + provider: string +} +const { provider } = Astro.props +const info: Record & { openaiCompatibility?: string } = + LLMS.providers.find(({ id }) => id === provider) as any +if (!info) { + throw new Error(`Provider ${provider} not found`) +} +const features: Record = { + seed: { + name: "Seed ignored", + }, + topP: { + name: "top_p ignored", + }, + logprobs: { + name: "logprobs (and top logprobs) ignored", + }, + topLogrobs: { + name: "Top logprobs ignored", + }, + tools: { + name: "Tools implemented as fallback tools automatically.", + }, +} +const oai = info.openaiCompatibility +const unsupported = Object.keys(info) + .sort() + .map((id) => ({ id, supported: info[id] })) + .filter(({ supported }) => supported === false) +--- + +{ + oai || unsupported?.length > 0 ? ( + <> +

Limitations

+
    + {!!oai && ( +
  • + Uses OpenAI compatibility layer +
    • + )} + {Object.keys(features) + .map((id) => ({ id, supported: info[id] })) + .filter(({ supported }) => supported === false) + .map(({ id }) => ( +
  • {features[id]?.name || id}
    • + ))} +
+ + ) : null +} diff --git a/docs/src/content/docs/getting-started/configuration.mdx b/docs/src/content/docs/getting-started/configuration.mdx index 294bf90359..79c4245214 100644 --- a/docs/src/content/docs/getting-started/configuration.mdx +++ b/docs/src/content/docs/getting-started/configuration.mdx @@ -10,6 +10,7 @@ import { FileTree } from "@astrojs/starlight/components" import { Steps } from "@astrojs/starlight/components" import { Tabs, TabItem } from "@astrojs/starlight/components" import { Image } from "astro:assets" +import LLMProviderFeatures from "../../../components/LLMProviderFeatures.astro" import lmSrc from "../../../assets/vscode-language-models.png" import lmAlt from "../../../assets/vscode-language-models.png.txt?raw" @@ -245,6 +246,8 @@ GENAISCRIPT_MODEL_SMALL=openai:gpt-4o-mini ::: + + ## GitHub Models The [GitHub Models](https://github.com/marketplace/models) provider, `github`, allows running models through the GitHub Marketplace. @@ -330,6 +333,8 @@ script({ }) ``` + + ## Azure OpenAI The [Azure OpenAI](https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#chat-completions) provider, `azure` uses the `AZURE_OPENAI_...` environment variables. @@ -477,6 +482,8 @@ The rest of the steps are the same: Find the deployment name and use it in your + + ## Azure AI Serverless Deployments You can deploy "serverless" models through [Azure AI Studio](https://ai.azure.com/) and pay as you go per token. @@ -587,6 +594,8 @@ AZURE_SERVERLESS_OPENAI_API_KEY=... + + ### Azure AI Models The `azure_serverless_models` provider supports non-OpenAI models deployed through the Azure AI Studio serverless deployments. @@ -680,7 +689,7 @@ AZURE_SERVERLESS_MODELS_API_KEY=...
  • -Find the deployment name and use it in your script, `model: "azure_serverless:deployment-id"`. +Find the deployment name and use it in your script, `model: "azure_serverless_models:deployment-id"`.
    • @@ -700,6 +709,8 @@ model3=key3 " ``` + + ## Google AI     The `google` provider allows you to use Google AI models. It gives you access @@ -760,6 +771,8 @@ script({ model: "google:gemini-1.5-pro-002" }) + + ## GitHub Copilot Chat Models If you have access to **GitHub Copilot Chat in Visual Studio Code**, @@ -855,6 +868,8 @@ script({ + + ## Hugging Face The `huggingface` provider allows you to use [Hugging Face Models](https://huggingface.co/models?other=text-generation-inference) using [Text Generation Inference](https://huggingface.co/docs/text-generation-inference/index). @@ -915,6 +930,8 @@ Some models may require a Pro account. ::: + + ## Mistral AI The `mistral` provider allows you to use [Mistral AI Models](https://mistral.ai/technology/#models) @@ -962,6 +979,9 @@ script({ + + + ## Alibaba Cloud The `alibaba` provider access the [Alibaba Cloud](https://www.alibabacloud.com/) models. @@ -1015,12 +1035,7 @@ script({ -:::note - -GenAIScript uses the [OpenAI compatibility](https://www.alibabacloud.com/help/en/model-studio/developer-reference/compatibility-of-openai-with-dashscope) layer -to access Alibaba. - -::: + ## Ollama @@ -1117,6 +1132,8 @@ docker run -d -v ollama:/root/.ollama -p 11434:11434 --name ollama ollama/ollama docker stop ollama && docker rm ollama ``` + + ## LMStudio The `lmstudio` provider connects to the [LMStudio](https://lmstudio.ai/) headless server. @@ -1184,6 +1201,8 @@ script({ Follow [this guide](https://huggingface.co/blog/yagilb/lms-hf) to load Hugging Face models into LMStudio. + + ## Jan The `jan` provider connects to the [Jan](https://jan.ai/) local server. @@ -1224,6 +1243,8 @@ If you change the default server URL, you can set the `JAN_API_BASE` environment JAN_API_BASE=http://localhost:1234/v1 ``` + + ## LocalAI [LocalAI](https://localai.io/) act as a drop-in replacement REST API that’s compatible @@ -1348,6 +1369,9 @@ This provider is experimental and may not work with all models. ::: + + + ## Model specific environment variables You can provide different environment variables diff --git a/packages/core/src/llms.json b/packages/core/src/llms.json index 66910aef87..1298252779 100644 --- a/packages/core/src/llms.json +++ b/packages/core/src/llms.json @@ -25,13 +25,16 @@ }, { "id": "anthropic", - "detail": "Anthropic models" + "detail": "Anthropic models", + "logprobs": false, + "topLogprobs": false }, { - "id": "googe", + "id": "google", "detail": "Google AI", "seed": false, - "tools": false + "tools": false, + "openaiCompatibility": "https://ai.google.dev/gemini-api/docs/openai" }, { "id": "huggingface", @@ -48,7 +51,8 @@ { "id": "ollama", "detail": "Ollama local model", - "logitBias": false + "logitBias": false, + "openaiCompatibility": "https://github.com/ollama/ollama/blob/main/docs/openai.md" }, { "id": "lmstudio", @@ -61,6 +65,7 @@ { "id": "alibaba", "detail": "Alibaba models", + "openaiCompatibility": "https://www.alibabacloud.com/help/en/model-studio/developer-reference/compatibility-of-openai-with-dashscope", "tools": false }, {