From 15cea16a5abdebc4aaf2b2cfef7db5594f092674 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Ba=CC=88uerle?= Date: Fri, 29 Dec 2023 12:08:56 +0100 Subject: [PATCH] docs: refine the documentation and make some style changes --- docs/intro.mdx | 154 +------------------------------------- docs/start.mdx | 179 +++++++++++++++++++++++++++++++++++++++++++++ src/css/custom.css | 12 +++ src/pages/faq.jsx | 2 +- 4 files changed, 196 insertions(+), 151 deletions(-) create mode 100644 docs/start.mdx diff --git a/docs/intro.mdx b/docs/intro.mdx index 9c9fb38..e837e2b 100644 --- a/docs/intro.mdx +++ b/docs/intro.mdx @@ -4,164 +4,18 @@ sidebar_position: 1 import CodeBlock from "@theme/CodeBlock"; -# Getting Started +# What is Zeno? Zeno is an **interactive AI evaluation platform** for exploring, discovering and reporting the performance of your models. Use it for any task and data type with Zeno's [modular views](/docs/views/) for everything from object detection to audio transcription and chatbot conversations. Zeno helps you move beyond aggregate metrics and spot-checking model outputs to develop a deep and quantitative understanding of how your model behaves. -To get an idea of Zeno's features, you can explore public projects and reports in [Zeno Hub](https://hub.zenoml.com). -For example, take a look at a [report evaluating how well LLMs like GPT-4 do on language translation](https://hub.zenoml.com/report/cabreraalex/GPT%20MT%20Benchmark%20Report), along with the [underlying data used to create the report.](https://hub.zenoml.com/project/cabreraalex/GPT%20MT%20Benchmark/explore). - :::tip - Head over to [Zeno Hub](https://hub.zenoml.com) to explore public Zeno projects and reports. +For example, take a look at a [report evaluating how well LLMs like GPT-4 do on language translation](https://hub.zenoml.com/report/cabreraalex/GPT%20MT%20Benchmark%20Report), along with the [underlying data used to create the report](https://hub.zenoml.com/project/cabreraalex/GPT%20MT%20Benchmark/explore). Check out the [Using Zeno](/docs/using-zeno/) docs for more information on how to use the Zeno interface. ::: -## Creating a Project - -To create your own projects and reports, first create an account on [Zeno Hub](https://hub.zenoml.com/signup). -After logging in to Zeno Hub, generate your API key by clicking on your profile at the top right to navigate to your [account page](https://hub.zenoml.com/account). - -Next, install the `zeno-client` Python package, which is used to upload new datasets and AI system outputs: - -```bash -pip install zeno-client -``` - -We can now create a client with out API key, and use it to create a project and upload data. The client API works with Pandas DataFrames, so we can create a sample DataFrame looking at text sentiment classification: - -```python -from zeno_client import ZenoClient -import pandas as pd - -client = ZenoClient("YOUR API KEY HERE") - -df = pd.DataFrame( - { - "text": [ - "I love this movie!", - "I hate this movie!", - "This movie is ok.", - ], - "label": ["positive", "negative", "neutral"], - } -) - -# Explicitly save the index as a column to upload. -df["id"] = df.index - -# Add any additional columns you want to do analysis across. -df["input length"] = df["text"].str.len() -``` - -Let's create a project for this task. **Projects** in Zeno are a base dataset and any number of AI system outputs, and are used to evaluate and compare model performance. Here we create a project and upload our base dataset. - -:::info -The `view` option can take a [`string`](/docs/views/existing) for one of the standard views or a [`dict`](/docs/views/spec/) with a custom view specification. - -If you just want to look at tabular metadata, you can omit `view` from `create_project` and `data_column` from `upload_dataset`. -::: - -```python -from zeno_client import ZenoClient, ZenoMetric - -... - -project = client.create_project( - name="Sentiment Classification", - view="text-classification", - metrics=[ - ZenoMetric(name="accuracy", type="mean", columns=["correct"]), - ] -) - -project.upload_dataset(df, id_column="id", data_column='text', label_column="label") -``` - -We named our project "Sentiment Classification" and specified that it is a text classification task. -Check out all supported data types and tasks [here](https://zenoml.com/docs/views/). -We also added an initial accuracy metric which takes the mean of the `correct` column, which will be present in the system outputs we upload later. - -Next, we can upload some system outputs to evaluate. Here we'll upload some fake predictions from a model: - -```python - -... - -df_system = pd.DataFrame( - { - "output": ["positive", "negative", "negative"], - } -) - -# Create an id column to match the base dataset. -df_system["id"] = df_system.index - -# Measure accuracy for each instance, which is averaged by the ZenoMetric above. -df_system["correct"] = (df_system["output"] == df["label"]).astype(int) - -proj.upload_system(df_system, name="System A", id_column="id", output_column="output") -``` - -You can now navigate to the project URL in Zeno Hub to see the uploaded data and metrics and start exploring your AI system's performance! - -#### Complete Example - -```python -from zeno_client import ZenoClient, ZenoMetric -import pandas as pd - -client = ZenoClient("YOUR API KEY HERE") - -df = pd.DataFrame( - { - "text": [ - "I love this movie!", - "I hate this movie!", - "This movie is ok.", - ], - "label": ["positive", "negative", "neutral"], - } -) - -# Explicitly save the index as a column to upload. -df["id"] = df.index - -# Add any additional columns you want to do analysis across. -df["input length"] = df["text"].str.len() - -project = client.create_project( - name="Sentiment Classification", - view="text-classification", - metrics=[ - ZenoMetric(name="accuracy", type="mean", columns=["correct"]), - ] -) - -project.upload_dataset(df, id_column="id", data_column='text', label_column="label") - -df_system = pd.DataFrame( - { - "output": ["positive", "negative", "negative"], - } -) - -# Create an id column to match the base dataset. -df_system["id"] = df_system.index - -# Measure accuracy for each instance, which is averaged by the ZenoMetric above. -df_system["correct"] = (df_system["output"] == df["label"]).astype(int) - -project.upload_system(df_system, name="System A", id_column="id", output_column="output") -``` - -## Quickstart with Zeno Build - -[Zeno Build](https://github.com/zeno-ml/zeno-build) is a Python library that makes it easy to set up Zeno projects for common AI and ML tasks. Check out some common Zeno Build notebooks: - -- [EleutherAI LM Evaluation Harness](https://github.com/zeno-ml/zeno-build/tree/main/examples/eleuther_harness) -- [🤗 OpenLLM Leaderboard](https://github.com/zeno-ml/zeno-build/tree/main/examples/open_llm_leaderboard) -- [Audio Transcription Bias](https://github.com/zeno-ml/zeno-build/tree/main/examples/transcription) +The best way to use Zeno is with your own data and models. +Learn how to create your [own projects](/docs/start)! diff --git a/docs/start.mdx b/docs/start.mdx new file mode 100644 index 0000000..e5c8488 --- /dev/null +++ b/docs/start.mdx @@ -0,0 +1,179 @@ +--- +sidebar_position: 2 +--- + +# Creating a Project + +Creating a new Zeno project is super easy. +All you need is a Zeno account and some data you want to upload and investigate. +We will go thruogh creating your first project step by step, but also have the complete code example right here for your reference: + +
+ Complete Example +
+ ```python + from zeno_client import ZenoClient, ZenoMetric + import pandas as pd + + client = ZenoClient("YOUR API KEY HERE") + + df = pd.DataFrame( + { + "id": [1, 2, 3], + "text": [ + "I love this movie!", + "I hate this movie!", + "This movie is ok.", + ], + "label": ["positive", "negative", "neutral"], + } + ) + + # Add any additional columns you want to do analysis across. + df["input length"] = df["text"].str.len() + + # Create a project. + project = client.create_project( + name="Sentiment Classification", + view="text-classification", + metrics=[ + ZenoMetric(name="accuracy", type="mean", columns=["correct"]), + ] + ) + + # Upload the data. + project.upload_dataset(df, id_column="id", data_column='text', label_column="label") + + # Create a system DataFrame. + df_system = pd.DataFrame( + { + "output": ["positive", "negative", "negative"], + } + ) + + # Create an id column to match the base dataset. + df_system["id"] = df_system.index + + # Measure accuracy for each instance, which is averaged by the ZenoMetric above. + df_system["correct"] = (df_system["output"] == df["label"]).astype(int) + + proj.upload_system(df_system, name="System A", id_column="id", output_column="output") + ``` + +
+
+ +## Zeno Account + +If you don't have a Zeno account already, create one on [Zeno Hub](https://hub.zenoml.com/signup). +After logging in to Zeno Hub, generate your API key by clicking on your profile at the top right to navigate to your [account page](https://hub.zenoml.com/account). + +## Data Upload + +We're uploading data directly from Python. +This makes it easy for you to start your evaluation right where you do your AI system development. + +### Zeno Client + +To get all the functions used to upload new datasets and AI system outputs, install the `zeno-client` Python package: + +```bash +pip install zeno-client +``` + +We can now initialize a client with our API key and use it to create a project and upload data. + +```python +from zeno_client import ZenoClient, ZenoMetric +import pandas as pd + +# Initialize a client with our API key. +client = ZenoClient("YOUR API KEY HERE") +``` + +:::tip +If you want to learn more about our Python client, read our [client API docs](/docs/python-client). +::: + +### Data Format + +Zeno takes any data that you provide in a **Pandas DataFrame**. +In this example, we look at text sentiment classification: + +```python +... + +# Put all data in a Pandas DataFrame +df = pd.DataFrame( + { + "id": [1, 2, 3], + "text": [ + "I love this movie!", + "I hate this movie!", + "This movie is ok.", + ], + "label": ["positive", "negative", "neutral"], + } +) + +# Add any additional columns you want to do analysis across. +df["input length"] = df["text"].str.len() +``` + +### Zeno Project + +**Projects** in Zeno are a base dataset and any number of AI system outputs used to evaluate and compare model performance. +Here we create a project and upload our base dataset. + +```python +... + +project = client.create_project( + name="Sentiment Classification", + view="text-classification", + metrics=[ + ZenoMetric(name="accuracy", type="mean", columns=["correct"]), + ] +) + +project.upload_dataset(df, id_column="id", data_column='text', label_column="label") +``` + +We named our project _Sentiment Classification_ and specified to use the _text_classification_ view. +We also added an _accuracy_ metric which takes the mean of the `correct` column that will be present in the system outputs we upload later. + +:::tip +If you want to learn more about Zeno's powerful `view` option and what to best use for your data, read our [instance view docs](/docs/views). +::: + +### System Outputs + +Next, we can upload some system outputs to evaluate. Here we'll upload some fake predictions from a model: + +```python +... + +df_system = pd.DataFrame( + { + "output": ["positive", "negative", "negative"], + } +) + +# Create an id column to match the base dataset. +df_system["id"] = df_system.index + +# Measure accuracy for each instance, which is averaged by the ZenoMetric above. +df_system["correct"] = (df_system["output"] == df["label"]).astype(int) + +proj.upload_system(df_system, name="System A", id_column="id", output_column="output") +``` + +You can now navigate to the project URL in Zeno Hub to see the uploaded data and metrics and start exploring your AI system's performance! + +## Quickstart with Zeno Build + +[Zeno Build](https://github.com/zeno-ml/zeno-build) is a Python project that contains a collection of example projects for common AI and ML tasks. Check out some common Zeno Build notebooks: + +- [EleutherAI LM Evaluation Harness](https://github.com/zeno-ml/zeno-build/tree/main/examples/eleuther_harness) +- [🤗 OpenLLM Leaderboard](https://github.com/zeno-ml/zeno-build/tree/main/examples/open_llm_leaderboard) +- [Audio Transcription Bias](https://github.com/zeno-ml/zeno-build/tree/main/examples/transcription) diff --git a/src/css/custom.css b/src/css/custom.css index bdf0d9a..c9226e8 100644 --- a/src/css/custom.css +++ b/src/css/custom.css @@ -15,6 +15,10 @@ --ifm-color-primary-light: #751ea9; --ifm-color-primary-lighter: #7a1fb1; --ifm-color-primary-lightest: #8a23c8; + --ifm-color-info-contrast-background: #f7f1fb; + --ifm-color-info-dark: #6a1b9a; + --ifm-color-success-contrast-background: #f7f1fb; + --ifm-color-success-dark: #6a1b9a; --ifm-navbar-background-color: #f5f5f5; @@ -36,6 +40,10 @@ --ifm-color-primary-light: #ffffff; --ifm-color-primary-lighter: #ffffff; --ifm-color-primary-lightest: #ffffff; + --ifm-color-info-contrast-background: #4a136c !important; + --ifm-color-info-dark: #8a23c8; + --ifm-color-success-contrast-background: #4a136c !important; + --ifm-color-success-dark: #8a23c8; --ifm-navbar-background-color: #4a136c; } @@ -302,3 +310,7 @@ hr { .sign-in:hover { background: var(--ifm-color-primary-lightest); } + +.alert a { + text-decoration: var(--ifm-link-decoration); +} diff --git a/src/pages/faq.jsx b/src/pages/faq.jsx index 6a507d0..c488115 100644 --- a/src/pages/faq.jsx +++ b/src/pages/faq.jsx @@ -54,7 +54,7 @@ export default function FAQ() { exploring public projects and reports {" "} to get an idea of how Zeno works. To create your own projects please follow the{" "} - Getting Started Guide{" "} + Getting Started Guide.{" "}

How do I create a new project?