diff --git a/.github/workflows/embeddings-comp-test.yml b/.github/workflows/embeddings-comp-test.yml deleted file mode 100644 index 05d0cf2ee..000000000 --- a/.github/workflows/embeddings-comp-test.yml +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright (C) 2024 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -name: EmbeddingsMicroService-test - -on: - # pull_request: - # branches: [main] - # types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped - # paths: - # - .github/workflows/mega-test.yml - # - comps/mega/** - workflow_dispatch: - -# If there is a new commit, the previous jobs will be canceled -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - EmbeddingsMicroService: - runs-on: aise-cluster - steps: - - name: Clean Up Working Directory - run: sudo rm -rf ${{github.workspace}}/* - - - name: Checkout out Repo - uses: actions/checkout@v4 - with: - ref: "refs/pull/${{ github.event.number }}/merge" - - - name: Run UT - env: - HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }} - run: | - cd .github/workflows/scripts - bash test_ut.sh --test_name embeddings - - - name: Run Workflow - env: - HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }} - run: | - cd tests/workflows - bash test_embeddings_comps.sh - - - name: Publish pipeline artifact - if: ${{ !cancelled() }} - uses: actions/upload-artifact@v4 - with: - path: ${{ github.workspace }}/.github/workflows/scripts/*.log diff --git a/.github/workflows/llms-comp-test.yml b/.github/workflows/llms-comp-test.yml deleted file mode 100644 index 67369707a..000000000 --- a/.github/workflows/llms-comp-test.yml +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright (C) 2024 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -name: LLMsMicroService-test - -on: - # pull_request: - # branches: [main] - # types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped - # paths: - # - .github/workflows/llms-comp-test.yml - # - comps/llms/** - workflow_dispatch: - -# If there is a new commit, the previous jobs will be canceled -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - LLMsMicroService: - runs-on: aise-cluster - steps: - - name: Clean Up Working Directory - run: sudo rm -rf ${{github.workspace}}/* - - - name: Checkout out Repo - uses: actions/checkout@v4 - with: - ref: "refs/pull/${{ github.event.number }}/merge" - - - name: Run UT - env: - HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }} - run: | - cd .github/workflows/scripts - bash test_ut.sh --test_name llms - - - name: Run Workflow - env: - HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }} - run: | - cd tests/workflows - bash test_llms_comps.sh - - - name: Publish pipeline artifact - if: ${{ !cancelled() }} - uses: actions/upload-artifact@v4 - with: - path: ${{ github.workspace }}/.github/workflows/scripts/*.log diff --git a/.github/workflows/microservice-test.yml b/.github/workflows/microservice-test.yml new file mode 100644 index 000000000..2eafd4174 --- /dev/null +++ b/.github/workflows/microservice-test.yml @@ -0,0 +1,87 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +name: MicroService-test + +on: + pull_request_target: + branches: [main] + types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped + paths: + - comps/** + - "!**.md" + - "!**.txt" + - .github/workflows/microservice.yml + +# If there is a new commit, the previous jobs will be canceled +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + job1: + name: Get-test-matrix + runs-on: ubuntu-latest + outputs: + run_matrix: ${{ steps.get-test-matrix.outputs.run_matrix }} + steps: + - name: Checkout out Repo + uses: actions/checkout@v4 + with: + ref: "refs/pull/${{ github.event.number }}/merge" + fetch-depth: 0 + - name: Get test matrix + id: get-test-matrix + run: | + set -xe + changed_files=$(git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} \ + | grep 'comps/' | grep -vE '*.md|*.txt') + services=$(printf '%s\n' "${changed_files[@]}" | grep '/' | cut -d'/' -f2 | sort -u) + run_matrix="{\"include\":[" + for service in ${services}; do + hardware="gaudi" # default hardware, set based on the changed files + run_matrix="${run_matrix}{\"service\":\"${service}\",\"hardware\":\"${hardware}\"}," + done + run_matrix=$run_matrix"]}" + echo "run_matrix=${run_matrix}" >> $GITHUB_OUTPUT + + Microservice-test: + needs: job1 + strategy: + matrix: ${{ fromJSON(needs.job1.outputs.run_matrix) }} + runs-on: ${{ matrix.hardware }} + continue-on-error: true + steps: + - name: Clean Up Working Directory + run: sudo rm -rf ${{github.workspace}}/* + + - name: Checkout out Repo + uses: actions/checkout@v4 + with: + ref: "refs/pull/${{ github.event.number }}/merge" + + - name: Run microservice test + env: + HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }} + service: ${{ matrix.service }} + hardware: ${{ matrix.hardware }} + run: | + cd tests + if [ -f test_${service}.sh ]; then timeout 10m bash test_${service}.sh; else echo "Test script not found, skip test!"; fi + + - name: Clean up container + env: + service: ${{ matrix.service }} + hardware: ${{ matrix.hardware }} + if: cancelled() || failure() + run: | + cid=$(docker ps -aq --filter "name=test-comps-*") + if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi + echo y | docker system prune + + - name: Publish pipeline artifact + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.service }}-${{ matrix.hardware }} + path: ${{ github.workspace }}/tests/*.log diff --git a/comps/asr/README.md b/comps/asr/README.md index 27f50f6f6..295683c79 100644 --- a/comps/asr/README.md +++ b/comps/asr/README.md @@ -2,43 +2,45 @@ ASR (Audio-Speech-Recognition) microservice helps users convert speech to text. When building a talking bot with LLM, users will need to convert their audio inputs (What they talk, or Input audio from other sources) to text, so the LLM is able to tokenize the text and generate an answer. This microservice is built for that conversion stage. -# ๐Ÿš€Start Microservice with Python +# ๐Ÿš€1. Start Microservice with Python (Option 1) To start the ASR microservice with Python, you need to first install python packages. -## Install Requirements +## 1.1 Install Requirements ```bash pip install -r requirements.txt ``` -## Start ASR Service with Python Script +## 1.2 Start ASR Service with Python Script ```bash python asr.py ``` -# ๐Ÿš€Start Microservice with Docker +# ๐Ÿš€2. Start Microservice with Docker (Option 2) Alternatively, you can also start the ASR microservice with Docker. -## Build Docker Image +## 2.1 Build Docker Image ```bash cd ../../ -docker build -t opea/gen-ai-comps:asr --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/asr/Dockerfile . +docker build -t opea/asr:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/asr/Dockerfile . ``` -## Run Docker with CLI +## 2.2 Run Docker with CLI ```bash -docker run -p 9099:9099 --network=host --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy opea/gen-ai-comps:asr +docker run -p 9099:9099 --network=host --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy opea/asr:latest ``` -# Test +# ๐Ÿš€3. Consume ASR Service You can use the following `curl` command to test whether the service is up. Notice that the first request can be slow because it needs to download the models. ```bash -curl http://localhost:9099/v1/audio/transcriptions -H "Content-Type: application/json" -d '{"url": "https://github.com/intel/intel-extension-for-transformers/raw/main/intel_extension_for_transformers/neural_chat/assets/audio/sample_2.wav"}' +curl http://localhost:9099/v1/audio/transcriptions \ + -H "Content-Type: application/json" \ + -d '{"url": "https://github.com/intel/intel-extension-for-transformers/raw/main/intel_extension_for_transformers/neural_chat/assets/audio/sample_2.wav"}' ``` diff --git a/comps/dataprep/redis/README.md b/comps/dataprep/redis/README.md index 785afd6f3..87564c584 100644 --- a/comps/dataprep/redis/README.md +++ b/comps/dataprep/redis/README.md @@ -1,18 +1,18 @@ # Dataprep Microservice with Redis -# ๐Ÿš€Start Microservice with Python +# ๐Ÿš€1. Start Microservice with Python๏ผˆOption 1๏ผ‰ -## Install Requirements +## 1.1 Install Requirements ```bash pip install -r requirements.txt ``` -## Start Redis Stack Server +## 1.2 Start Redis Stack Server Please refer to this [readme](../../../vectorstores/langchain/redis/README.md). -## Setup Environment Variables +## 1.3 Setup Environment Variables ```bash export REDIS_URL="redis://${your_ip}:6379" @@ -22,7 +22,7 @@ export LANGCHAIN_API_KEY=${your_langchain_api_key} export LANGCHAIN_PROJECT="opea/gen-ai-comps:dataprep" ``` -## Start Document Preparation Microservice for Redis with Python Script +## 1.4 Start Document Preparation Microservice for Redis with Python Script Start document preparation microservice for Redis with below command. @@ -30,38 +30,49 @@ Start document preparation microservice for Redis with below command. python prepare_doc_redis.py ``` -# ๐Ÿš€Start Microservice with Docker +# ๐Ÿš€2. Start Microservice with Docker (Option 2) -## Build Docker Image +## 2.1 Start Redis Stack Server -```bash -cd ../../../../ -docker build -t opea/dataprep-redis:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/redis/docker/Dockerfile . -``` +Please refer to this [readme](../../../vectorstores/langchain/redis/README.md). -## Run Docker with CLI +## 2.2 Setup Environment Variables ```bash export REDIS_URL="redis://${your_ip}:6379" export INDEX_NAME=${your_index_name} export LANGCHAIN_TRACING_V2=true export LANGCHAIN_API_KEY=${your_langchain_api_key} -export LANGCHAIN_PROJECT="opea/gen-ai-comps:dataprep" +export LANGCHAIN_PROJECT="opea/dataprep" +``` + +## 2.3 Build Docker Image -docker run -d --name="dataprep-redis-server" -p 6007:6007 --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e REDIS_URL=$REDIS_URL -e INDEX_NAME=$INDEX_NAME opea/dataprep-redis:latest +```bash +cd ../../../../ +docker build -t opea/dataprep-redis:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/redis/docker/Dockerfile . +``` + +## 2.4 Run Docker with CLI (Option A) + +```bash +docker run -d --name="dataprep-redis-server" -p 6007:6007 --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e REDIS_URL=$REDIS_URL -e INDEX_NAME=$INDEX_NAME -e TEI_ENDPOINT=$TEI_ENDPOINT opea/dataprep-redis:latest ``` -## Run Docker with Docker Compose +## 2.5 Run with Docker Compose (Option B) ```bash cd comps/dataprep/redis/docker docker compose -f docker-compose-dataprep-redis.yaml up -d ``` -# Invoke Microservice +# ๐Ÿš€3. Consume Microservice Once document preparation microservice for Redis is started, user can use below command to invoke the microservice to convert the document to embedding and save to the database. ```bash -curl -X POST -H "Content-Type: application/json" -d '{"path":"/path/to/document"}' http://localhost:6007/v1/dataprep +curl -X POST \ + -H "Content-Type: application/json" \ + -d '{"path":"/path/to/document"}' \ + http://localhost:6007/v1/dataprep ``` diff --git a/comps/embeddings/README.md b/comps/embeddings/README.md index c8e6964fd..bcfdfffe3 100644 --- a/comps/embeddings/README.md +++ b/comps/embeddings/README.md @@ -14,7 +14,7 @@ Key Features: Users are albe to configure and build embedding-related services according to their actual needs. -# ๐Ÿš€Start Microservice with Python +# ๐Ÿš€1. Start Microservice with Python (Option 1) Currently, we provide two ways to implement the embedding service: @@ -24,19 +24,17 @@ Currently, we provide two ways to implement the embedding service: For both of the implementations, you need to install requirements first. -## Install Requirements +## 1.1 Install Requirements ```bash pip install -r langchain/requirements.txt ``` -## Start Embedding Service with Local Model +## 1.2 Start Embedding Service -```bash -python local_embedding.py -``` +You can select one of following ways to start the embedding service: -## Start Embedding Service with TEI +### Start Embedding Service with TEI First, you need to start a TEI service. @@ -67,31 +65,64 @@ export LANGCHAIN_PROJECT="opea/gen-ai-comps:embeddings" python embedding_tei_gaudi.py ``` -# ๐Ÿš€Start Microservice with Docker +### Start Embedding Service with Local Model + +```bash +cd langchain +python local_embedding.py +``` + +# ๐Ÿš€2. Start Microservice with Docker (Optional 2) + +## 2.1 Start Embedding Service with TEI + +First, you need to start a TEI service. + +```bash +your_port=8090 +model="BAAI/bge-large-en-v1.5" +revision="refs/pr/5" +docker run -p $your_port:80 -v ./data:/data --name tei_server -e http_proxy=$http_proxy -e https_proxy=$https_proxy --pull always ghcr.io/huggingface/text-embeddings-inference:cpu-1.2 --model-id $model --revision $revision +``` + +Then you need to test your TEI service using the following commands: + +```bash +curl localhost:$your_port/embed \ + -X POST \ + -d '{"inputs":"What is Deep Learning?"}' \ + -H 'Content-Type: application/json' +``` + +Export the `TEI_EMBEDDING_ENDPOINT` for later usage: + +```bash +export TEI_EMBEDDING_ENDPOINT="http://localhost:$yourport" +``` -## Build Docker Image +## 2.2 Build Docker Image ```bash cd ../../ docker build -t opea/embedding-tei:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/langchain/docker/Dockerfile . ``` -## Run Docker with CLI +## 2.3 Run Docker with CLI ```bash docker run -d --name="embedding-tei-server" -p 6000:6000 --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e TEI_EMBEDDING_ENDPOINT=$TEI_EMBEDDING_ENDPOINT opea/embedding-tei:latest ``` -## Run Docker with Docker Compose +## 2.4 Run Docker with Docker Compose ```bash cd docker -docker compose -f docker_compose.yaml up -d +docker compose -f docker_compose_embedding.yaml up -d ``` -# ๐Ÿš€Consume Embedding Service +# ๐Ÿš€3. Consume Embedding Service -## Check Service Status +## 3.1 Check Service Status ```bash curl http://localhost:6000/v1/health_check\ @@ -99,7 +130,7 @@ curl http://localhost:6000/v1/health_check\ -H 'Content-Type: application/json' ``` -## Consume Embedding Service +## 3.2 Consume Embedding Service ```bash curl http://localhost:6000/v1/embeddings\ diff --git a/comps/guardrails/README.md b/comps/guardrails/README.md index e3f1705b2..e3fe84ea5 100644 --- a/comps/guardrails/README.md +++ b/comps/guardrails/README.md @@ -20,30 +20,30 @@ Any content that is detected in the following categories is determined as unsafe - Regulated or Controlled Substances - Suicide & Self Harm -# ๐Ÿš€Start Microservice with Python +# ๐Ÿš€1. Start Microservice with Python (Option 1) To start the Guardrails microservice, you need to install python packages first. -## Install Requirements +## 1.1 Install Requirements ```bash pip install -r requirements.txt ``` -## Start TGI Gaudi Service Manually +## 1.2 Start TGI Gaudi Service ```bash export HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token} export LANGCHAIN_TRACING_V2=true export LANGCHAIN_API_KEY=${your_langchain_api_key} -export LANGCHAIN_PROJECT="opea/gen-ai-comps:gaurdrails" +export LANGCHAIN_PROJECT="opea/gaurdrails" volume=$PWD/data model_id="meta-llama/LlamaGuard-7b" docker pull ghcr.io/huggingface/tgi-gaudi:1.2.1 docker run -p 8088:80 -v $volume:/data --runtime=habana -e HABANA_VISIBLE_DEVICES=all -e OMPI_MCA_btl_vader_single_copy_mechanism=none --cap-add=sys_nice --ipc=host -e HTTPS_PROXY=$https_proxy -e HTTP_PROXY=$https_proxy ghcr.io/huggingface/tgi-gaudi:1.2.1 --model-id $model_id ``` -## Verify the TGI Gaudi Service +## 1.3 Verify the TGI Gaudi Service ```bash curl 127.0.0.1:8088/generate \ @@ -52,18 +52,18 @@ curl 127.0.0.1:8088/generate \ -H 'Content-Type: application/json' ``` -## Start Guardrails Service with Python Script +## 1.4 Start Guardrails Service ```bash export SAFETY_GUARD_ENDPOINT="http://${your_ip}:8088" python langchain/guardrails_tgi_gaudi.py ``` -# ๐Ÿš€Start Microservice with Docker +# ๐Ÿš€2. Start Microservice with Docker (Option 2) If you start an Guardrails microservice with docker, the `docker_compose_guardrails.yaml` file will automatically start a TGI gaudi service with docker. -## Setup Environment Variables +## 2.1 Setup Environment Variables In order to start TGI and LLM services, you need to setup the following environment variables first. @@ -76,29 +76,29 @@ export LANGCHAIN_API_KEY=${your_langchain_api_key} export LANGCHAIN_PROJECT="opea/gen-ai-comps:gaurdrails" ``` -## Build Docker Image +## 2.2 Build Docker Image ```bash cd ../../ docker build -t opea/guardrails-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/guardrails/langchain/docker/Dockerfile . ``` -## Run Docker with CLI +## 2.3 Run Docker with CLI ```bash docker run -d --name="guardrails-tgi-server" -p 9090:9090 --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e SAFETY_GUARD_ENDPOINT=$SAFETY_GUARD_ENDPOINT -e HUGGINGFACEHUB_API_TOKEN=$HUGGINGFACEHUB_API_TOKEN opea/guardrails-tgi:latest ``` -## Run Docker with Docker Compose +## 2.4 Run Docker with Docker Compose ```bash cd langchain/docker docker compose -f docker_compose_guardrails.yaml up -d ``` -# ๐Ÿš€Consume Guardrails Service +# ๐Ÿš€3. Consume Guardrails Service -## Check Service Status +## 3.1 Check Service Status ```bash curl http://localhost:9090/v1/health_check\ @@ -106,7 +106,7 @@ curl http://localhost:9090/v1/health_check\ -H 'Content-Type: application/json' ``` -## Consume Guardrails Service +## 3.2 Consume Guardrails Service ```bash curl http://localhost:9090/v1/guardrails\ diff --git a/comps/llms/README.md b/comps/llms/README.md index 1408fecbc..95c5fac2f 100644 --- a/comps/llms/README.md +++ b/comps/llms/README.md @@ -6,17 +6,17 @@ A prerequisite for using this microservice is that users must have a Text Genera Overall, this microservice offers a streamlined way to integrate large language model inference into applications, requiring minimal setup from the user beyond initiating a TGI service and configuring the necessary environment variables. This allows for the seamless processing of queries and documents to generate intelligent, context-aware responses. -# ๐Ÿš€Start Microservice with Python +# ๐Ÿš€1. Start Microservice with Python (Option 1) To start the LLM microservice, you need to install python packages first. -## Install Requirements +## 1.1 Install Requirements ```bash pip install -r requirements.txt ``` -## Start TGI Service Manually +## 1.2 Start TGI Service ```bash export HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token} @@ -26,7 +26,7 @@ export LANGCHAIN_PROJECT="opea/gen-ai-comps:llms" docker run -p 8008:80 -v ./data:/data --name tgi_service --shm-size 1g ghcr.io/huggingface/text-generation-inference:1.4 --model-id ${your_hf_llm_model} ``` -## Verify the TGI Service +## 1.3 Verify the TGI Service ```bash curl http://${your_ip}:8008/generate \ @@ -35,18 +35,18 @@ curl http://${your_ip}:8008/generate \ -H 'Content-Type: application/json' ``` -## Start LLM Service with Python Script +## 1.4 Start LLM Service ```bash export TGI_LLM_ENDPOINT="http://${your_ip}:8008" python text-generation/tgi/llm.py ``` -# ๐Ÿš€Start Microservice with Docker +# ๐Ÿš€2. Start Microservice with Docker (Option 2) If you start an LLM microservice with docker, the `docker_compose_llm.yaml` file will automatically start a TGI service with docker. -## Setup Environment Variables +## 2.1 Setup Environment Variables In order to start TGI and LLM services, you need to setup the following environment variables first. @@ -56,32 +56,39 @@ export TGI_LLM_ENDPOINT="http://${your_ip}:8008" export LLM_MODEL_ID=${your_hf_llm_model} export LANGCHAIN_TRACING_V2=true export LANGCHAIN_API_KEY=${your_langchain_api_key} -export LANGCHAIN_PROJECT="opea/gen-ai-comps:llms" +export LANGCHAIN_PROJECT="opea/llms" ``` -## Build Docker Image +## 2.2 Build Docker Image ```bash cd ../../ docker build -t opea/llm-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/text-generation/tgi/Dockerfile . ``` -## Run Docker with CLI +To start a docker container, you have two options: + +- A. Run Docker with CLI +- B. Run Docker with Docker Compose + +You can choose one as needed. + +## 2.3 Run Docker with CLI (Option A) ```bash docker run -d --name="llm-tgi-server" -p 9000:9000 --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e TGI_LLM_ENDPOINT=$TGI_LLM_ENDPOINT -e HUGGINGFACEHUB_API_TOKEN=$HUGGINGFACEHUB_API_TOKEN opea/llm-tgi:latest ``` -## Run Docker with Docker Compose +## 2.4 Run Docker with Docker Compose (Option B) ```bash cd text-generation/tgi docker compose -f docker_compose_llm.yaml up -d ``` -# ๐Ÿš€Consume LLM Service +# ๐Ÿš€3. Consume LLM Service -## Check Service Status +## 3.1 Check Service Status ```bash curl http://${your_ip}:9000/v1/health_check\ @@ -89,7 +96,7 @@ curl http://${your_ip}:9000/v1/health_check\ -H 'Content-Type: application/json' ``` -## Consume LLM Service +## 3.2 Consume LLM Service You can set the following model parameters according to your actual needs, such as `max_new_tokens`, `streaming`. diff --git a/comps/reranks/README.md b/comps/reranks/README.md index 8b66de934..46493ca72 100644 --- a/comps/reranks/README.md +++ b/comps/reranks/README.md @@ -2,30 +2,30 @@ The Reranking Microservice, fueled by reranking models, stands as a straightforward yet immensely potent tool for semantic search. When provided with a query and a collection of documents, reranking swiftly indexes the documents based on their semantic relevance to the query, arranging them from most to least pertinent. This microservice significantly enhances overall accuracy. In a text retrieval system, either a dense embedding model or a sparse lexical search index is often employed to retrieve relevant text documents based on the input. However, a reranking model can further refine this process by rearranging potential candidates into a final, optimized order. -# ๐Ÿš€Start Microservice with Python +# ๐Ÿš€1. Start Microservice with Python (Option 1) To start the Reranking microservice, you must first install the required python packages. -## Install Requirements +## 1.1 Install Requirements ```bash pip install -r requirements.txt ``` -## Start TEI Service Manually +## 1.2 Start TEI Service ```bash export HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token} export LANGCHAIN_TRACING_V2=true export LANGCHAIN_API_KEY=${your_langchain_api_key} -export LANGCHAIN_PROJECT="opea/gen-ai-comps:reranks" +export LANGCHAIN_PROJECT="opea/reranks" model=BAAI/bge-reranker-large revision=refs/pr/4 volume=$PWD/data docker run -d -p 6060:80 -v $volume:/data -e http_proxy=$http_proxy -e https_proxy=$https_proxy --pull always ghcr.io/huggingface/text-embeddings-inference:cpu-1.2 --model-id $model --revision $revision ``` -## Verify the TEI Service +## 1.3 Verify the TEI Service ```bash curl 127.0.0.1:6060/rerank \ @@ -34,42 +34,57 @@ curl 127.0.0.1:6060/rerank \ -H 'Content-Type: application/json' ``` -## Start Reranking Service with Python Script +## 1.4 Start Reranking Service with Python Script ```bash export TEI_RERANKING_ENDPOINT="http://${your_ip}:6060" python reranking_tei_xeon.py ``` -# ๐Ÿš€Start Microservice with Docker +# ๐Ÿš€2. Start Microservice with Docker (Option 2) If you start an Reranking microservice with docker, the `docker_compose_reranking.yaml` file will automatically start a TEI service with docker. -## Setup Environment Variables +## 2.1 Setup Environment Variables -## Build Docker Image +```bash +export HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token} +export LANGCHAIN_TRACING_V2=true +export LANGCHAIN_API_KEY=${your_langchain_api_key} +export LANGCHAIN_PROJECT="opea/reranks" +export TEI_RERANKING_ENDPOINT="http://${your_ip}:8808" +``` + +## 2.2 Build Docker Image ```bash cd ../../ docker build -t opea/reranking-tei:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/reranks/langchain/docker/Dockerfile . ``` -## Run Docker with CLI +To start a docker container, you have two options: + +- A. Run Docker with CLI +- B. Run Docker with Docker Compose + +You can choose one as needed. + +## 2.3 Run Docker with CLI (Option A) ```bash -docker run -d --name="reranking-tei-server" -p 8000:8000 --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e TEI_RERANKING_ENDPOINT=$TEI_RERANKING_ENDPOINT -e HUGGINGFACEHUB_API_TOKEN=$HUGGINGFACEHUB_API_TOKEN opea/reranking-tei:latest +docker run -d --name="reranking-tei-server" -p 8000:8000 --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e TEI_RERANKING_ENDPOINT=$TEI_RERANKING_ENDPOINT -e HUGGINGFACEHUB_API_TOKEN=$HUGGINGFACEHUB_API_TOKEN -e LANGCHAIN_API_KEY=$LANGCHAIN_API_KEY opea/reranking-tei:latest ``` -## Run Docker with Docker Compose +## 2.4 Run Docker with Docker Compose (Option B) ```bash cd langchain/docker docker compose -f docker_compose_reranking.yaml up -d ``` -# ๐Ÿš€Consume Reranking Service +# ๐Ÿš€3. Consume Reranking Service -## Check Service Status +## 3.1 Check Service Status ```bash curl http://localhost:8000/v1/health_check \ @@ -77,7 +92,7 @@ curl http://localhost:8000/v1/health_check \ -H 'Content-Type: application/json' ``` -## Consume Reranking Service +## 3.2 Consume Reranking Service ```bash curl http://localhost:8000/v1/reranking \ diff --git a/comps/retrievers/README.md b/comps/retrievers/README.md index ac95f0b43..bad388f1f 100644 --- a/comps/retrievers/README.md +++ b/comps/retrievers/README.md @@ -6,17 +6,38 @@ The service primarily utilizes similarity measures in vector space to rapidly re Overall, this microservice provides robust backend support for applications requiring efficient similarity searches, playing a vital role in scenarios such as recommendation systems, information retrieval, or any other context where precise measurement of document similarity is crucial. -# ๐Ÿš€Start Microservice with Python +# ๐Ÿš€1. Start Microservice with Python (Option 1) To start the retriever microservice, you must first install the required python packages. -## Install Requirements +## 1.1 Install Requirements ```bash pip install -r requirements.txt ``` -## Setup VectorDB Service +## 1.2 Start TEI Service + +```bash +export LANGCHAIN_TRACING_V2=true +export LANGCHAIN_API_KEY=${your_langchain_api_key} +export LANGCHAIN_PROJECT="opea/retriever" +model=BAAI/bge-base-en-v1.5 +revision=refs/pr/4 +volume=$PWD/data +docker run -d -p 6060:80 -v $volume:/data -e http_proxy=$http_proxy -e https_proxy=$https_proxy --pull always ghcr.io/huggingface/text-embeddings-inference:cpu-1.2 --model-id $model --revision $revision +``` + +## 1.3 Verify the TEI Service + +```bash +curl 127.0.0.1:6060/rerank \ + -X POST \ + -d '{"query":"What is Deep Learning?", "texts": ["Deep Learning is not...", "Deep learning is..."]}' \ + -H 'Content-Type: application/json' +``` + +## 1.4 Setup VectorDB Service You need to setup your own VectorDB service (Redis in this example), and ingest your knowledge documents into the vector database. @@ -27,48 +48,57 @@ Remember to ingest data into it manually. docker run -d --name="redis-vector-db" -p 6379:6379 -p 8001:8001 redis/redis-stack:7.2.0-v9 ``` -## Setup Environment Variables +## 1.5 Start Retriever Service ```bash +export TEI_EMBEDDING_ENDPOINT="http://${your_ip}:6060" +python langchain/retriever_redis.py +``` + +# ๐Ÿš€2. Start Microservice with Docker (Option 2) + +## 2.1 Setup Environment Variables + +```bash +export RETRIEVE_MODEL_ID="BAAI/bge-base-en-v1.5" export REDIS_URL="redis://${your_ip}:6379" export INDEX_NAME=${your_index_name} -export TEI_EMBEDDING_ENDPOINT=${your_embedding_endpoint} +export TEI_EMBEDDING_ENDPOINT="http://${your_ip}:6060" export LANGCHAIN_TRACING_V2=true export LANGCHAIN_API_KEY=${your_langchain_api_key} -export LANGCHAIN_PROJECT="opea/gen-ai-comps:retrievers" +export LANGCHAIN_PROJECT="opea/retrievers" ``` -## Start Retriever Service with Local Model +## 2.2 Build Docker Image ```bash -python langchain/retriever_redis.py +cd ../../ +docker build -t opea/retriever-redis:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/langchain/docker/Dockerfile . ``` -# ๐Ÿš€Start Microservice with Docker +To start a docker container, you have two options: -## Build Docker Image +- A. Run Docker with CLI +- B. Run Docker with Docker Compose -```bash -cd ../../ -docker build -t opea/retriever-redis:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/langchain/docker/Dockerfile . -``` +You can choose one as needed. -## Run Docker with CLI +## 2.3 Run Docker with CLI (Option A) ```bash -docker run -d --name="retriever-redis-server" -p 7000:7000 --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e REDIS_URL=$REDIS_URL -e INDEX_NAME=$INDEX_NAME opea/retriever-redis:latest +docker run -d --name="retriever-redis-server" -p 7000:7000 --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e REDIS_URL=$REDIS_URL -e INDEX_NAME=$INDEX_NAME -e TEI_EMBEDDING_ENDPOINT=$TEI_EMBEDDING_ENDPOINT opea/retriever-redis:latest ``` -## Run Docker with Docker Compose +## 2.4 Run Docker with Docker Compose (Option B) ```bash cd langchain/docker docker compose -f docker_compose_retriever.yaml up -d ``` -# ๐Ÿš€Consume Retriever Service +# ๐Ÿš€3. Consume Retriever Service -## Check Service Status +## 3.1 Check Service Status ```bash curl http://localhost:7000/v1/health_check \ @@ -76,7 +106,7 @@ curl http://localhost:7000/v1/health_check \ -H 'Content-Type: application/json' ``` -## Consume Embedding Service +## 3.2 Consume Embedding Service To consume the Retriever Microservice, you need to generate a mock embedding vector of length 768 in Python script: diff --git a/comps/retrievers/langchain/docker/docker_compose_retriever.yaml b/comps/retrievers/langchain/docker/docker_compose_retriever.yaml index ba3e409c1..df32acbd0 100644 --- a/comps/retrievers/langchain/docker/docker_compose_retriever.yaml +++ b/comps/retrievers/langchain/docker/docker_compose_retriever.yaml @@ -15,6 +15,15 @@ version: "3.8" services: + tei_xeon_service: + image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.2 + container_name: tei-xeon-server + ports: + - "6060:80" + volumes: + - "./data:/data" + shm_size: 1g + command: --model-id ${RETRIEVE_MODEL_ID} retriever: image: opea/retriever-redis:latest container_name: retriever-redis-server @@ -26,7 +35,6 @@ services: https_proxy: ${https_proxy} REDIS_URL: ${REDIS_URL} INDEX_NAME: ${INDEX_NAME} - TEI_EMBEDDING_ENDPOINT: ${TEI_EMBEDDING_ENDPOINT} LANGCHAIN_API_KEY: ${LANGCHAIN_API_KEY} restart: unless-stopped diff --git a/comps/tts/README.md b/comps/tts/README.md index 34f8f2106..b0aa32456 100644 --- a/comps/tts/README.md +++ b/comps/tts/README.md @@ -2,43 +2,45 @@ TTS (Text-To-Speech) microservice helps users convert text to speech. When building a talking bot with LLM, users might need an LLM generated answer in audio format. This microservice is built for that conversion stage. -# ๐Ÿš€Start Microservice with Python +# ๐Ÿš€1. Start Microservice with Python (Option 1) To start the TTS microservice, you need to first install python packages. -## Install Requirements +## 1.1 Install Requirements ```bash pip install -r requirements.txt ``` -## Start TTS Service with Python Script +## 1.2 Start TTS Service with Python Script ```bash python tts.py ``` -# ๐Ÿš€Start Microservice with Docker +# ๐Ÿš€2. Start Microservice with Docker (Option 2) Alternatively, you can start the ASR microservice with Docker. -## Build Docker Image +## 2.1 Build Docker Image ```bash cd ../../ -docker build -t opea/gen-ai-comps:tts --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/tts/Dockerfile . +docker build -t opea/tts:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/tts/Dockerfile . ``` -## Run Docker with CLI +## 2.2 Run Docker with CLI ```bash -docker run -p 9999:9999 --network=host --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy opea/gen-ai-comps:tts +docker run -p 9999:9999 --network=host --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy opea/tts:latest ``` -# Test +# ๐Ÿš€3. Consume TTS Service You can use the following `curl` command to test whether the service is up. Notice that the first request can be slow because it needs to download the models. ```bash -curl http://localhost:9999/v1/audio/speech -H "Content-Type: application/json" -d '{"text":"Hello there."}' +curl http://localhost:9999/v1/audio/speech \ + -H "Content-Type: application/json" \ + -d '{"text":"Hello there."}' ``` diff --git a/tests/test_embeddings.sh b/tests/test_embeddings.sh new file mode 100644 index 000000000..65f29b023 --- /dev/null +++ b/tests/test_embeddings.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -xe + +WORKPATH=$(dirname "$PWD") +ip_address=$(hostname -I | awk '{print $1}') + +function build_docker_images() { + cd $WORKPATH + echo $(pwd) + docker build --no-cache -t opea/embedding-tei:comps -f comps/embeddings/langchain/docker/Dockerfile . +} + +function start_service() { + tei_endpoint=5001 + model="BAAI/bge-large-en-v1.5" + revision="refs/pr/5" + unset http_proxy + docker run -d --name="test-comps-embedding-tei-endpoint" -p $tei_endpoint:80 -v ./data:/data --pull always ghcr.io/huggingface/text-embeddings-inference:cpu-1.2 --model-id $model --revision $revision + export TEI_EMBEDDING_ENDPOINT="http://${ip_address}:${tei_endpoint}" + tei_service_port=5002 + docker run -d --name="test-comps-embedding-tei-server" -e http_proxy=$http_proxy -e https_proxy=$https_proxy -p ${tei_service_port}:6000 --ipc=host -e TEI_EMBEDDING_ENDPOINT=$TEI_EMBEDDING_ENDPOINT opea/embedding-tei:comps + sleep 3m +} + +function validate_microservice() { + tei_service_port=5002 + http_proxy="" curl http://${ip_address}:$tei_service_port/v1/embeddings \ + -X POST \ + -d '{"text":"What is Deep Learning?"}' \ + -H 'Content-Type: application/json' +} + +function stop_docker() { + cid=$(docker ps -aq --filter "name=test-comps-embedding-*") + if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi +} + +function main() { + + stop_docker + + build_docker_images + start_service + + validate_microservice + + stop_docker + echo y | docker system prune + +} + +main diff --git a/tests/test_llms.sh b/tests/test_llms.sh new file mode 100644 index 000000000..0a29932f7 --- /dev/null +++ b/tests/test_llms.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -xe + +WORKPATH=$(dirname "$PWD") +ip_address=$(hostname -I | awk '{print $1}') + +function build_docker_images() { + cd $WORKPATH + docker build --no-cache -t opea/llm-tgi:comps -f comps/llms/text-generation/tgi/Dockerfile . +} + +function start_service() { + tgi_endpoint_port=5004 + export your_hf_llm_model="Intel/neural-chat-7b-v3-3" + # Remember to set HUGGINGFACEHUB_API_TOKEN before invoking this test! + export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} + docker run -d --name="test-comps-llm-tgi-endpoint" -p $tgi_endpoint_port:80 -v ./data:/data --shm-size 1g ghcr.io/huggingface/text-generation-inference:1.4 --model-id ${your_hf_llm_model} + export TGI_LLM_ENDPOINT="http://${ip_address}:${tgi_endpoint_port}" + + tei_service_port=5005 + unset http_proxy + docker run -d --name="test-comps-llm-tgi-server" -p ${tei_service_port}:9000 --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e TGI_LLM_ENDPOINT=$TGI_LLM_ENDPOINT -e HUGGINGFACEHUB_API_TOKEN=$HUGGINGFACEHUB_API_TOKEN opea/llm-tgi:comps + + # check whether tgi is fully ready + n=0 + until [[ "$n" -ge 100 ]] || [[ $ready == true ]]; do + docker logs test-comps-llm-tgi-endpoint > test-comps-llm-tgi-endpoint.log + n=$((n+1)) + if grep -q Connected test-comps-llm-tgi-endpoint.log; then + break + fi + sleep 5s + done + # rm -f test-comps-llm-tgi-endpoint.log + sleep 5s +} + +function validate_microservice() { + tei_service_port=5005 + http_proxy="" curl http://${ip_address}:${tei_service_port}/v1/chat/completions \ + -X POST \ + -d '{"query":"What is Deep Learning?"}' \ + -H 'Content-Type: application/json' +} + +function stop_docker() { + cid=$(docker ps -aq --filter "name=test-comps-llm-*") + if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi +} + +function main() { + + stop_docker + + build_docker_images + start_service + + validate_microservice + + stop_docker + echo y | docker system prune + +} + +main diff --git a/tests/test_reranks.sh b/tests/test_reranks.sh new file mode 100644 index 000000000..fd8ff7fb9 --- /dev/null +++ b/tests/test_reranks.sh @@ -0,0 +1,57 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -xe + +WORKPATH=$(dirname "$PWD") +ip_address=$(hostname -I | awk '{print $1}') +function build_docker_images() { + cd $WORKPATH + docker build --no-cache -t opea/reranking-tei:comps -f comps/reranks/langchain/docker/Dockerfile . +} + +function start_service() { + tei_endpoint=5006 + # Remember to set HUGGINGFACEHUB_API_TOKEN before invoking this test! + export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} + model=BAAI/bge-reranker-large + revision=refs/pr/4 + volume=$PWD/data + docker run -d --name="test-comps-reranking-tei-endpoint" -p $tei_endpoint:80 -v $volume:/data --pull always ghcr.io/huggingface/text-embeddings-inference:cpu-1.2 --model-id $model --revision $revision + + export TEI_RERANKING_ENDPOINT="http://${ip_address}:${tei_endpoint}" + tei_service_port=5007 + unset http_proxy + docker run -d --name="test-comps-reranking-tei-server" -p ${tei_service_port}:8000 --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e TEI_RERANKING_ENDPOINT=$TEI_RERANKING_ENDPOINT -e HUGGINGFACEHUB_API_TOKEN=$HUGGINGFACEHUB_API_TOKEN opea/reranking-tei:comps + sleep 1m +} + +function validate_microservice() { + tei_service_port=5007 + http_proxy="" curl http://${ip_address}:${tei_service_port}/v1/reranking\ + -X POST \ + -d '{"initial_query":"What is Deep Learning?", "retrieved_docs": [{"text":"Deep Learning is not..."}, {"text":"Deep learning is..."}]}' \ + -H 'Content-Type: application/json' +} + +function stop_docker() { + cid=$(docker ps -aq --filter "name=test-comps-rerank*") + if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi +} + +function main() { + + stop_docker + + build_docker_images + start_service + + validate_microservice + + stop_docker + echo y | docker system prune + +} + +main diff --git a/tests/test_retrievers.sh b/tests/test_retrievers.sh new file mode 100644 index 000000000..97f8ca7a3 --- /dev/null +++ b/tests/test_retrievers.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -xe + +WORKPATH=$(dirname "$PWD") +ip_address=$(hostname -I | awk '{print $1}') +function build_docker_images() { + cd $WORKPATH + docker build --no-cache -t opea/retriever-redis:comps -f comps/retrievers/langchain/docker/Dockerfile . +} + +function start_service() { + # tei endpoint + tei_endpoint=5008 + model="BAAI/bge-large-en-v1.5" + revision="refs/pr/5" + docker run -d --name="test-comps-retriever-tei-endpoint" -p $tei_endpoint:80 -v ./data:/data --pull always ghcr.io/huggingface/text-embeddings-inference:cpu-1.2 --model-id $model --revision $revision + export TEI_EMBEDDING_ENDPOINT="http://${ip_address}:${tei_endpoint}" + + # redis retriever + export REDIS_URL="redis://${ip_address}:6379" + export INDEX_NAME="rag-redis" + retriever_port=5009 + unset http_proxy + docker run -d --name="test-comps-retriever-redis-server" -p ${retriever_port}:7000 --ipc=host -e TEI_EMBEDDING_ENDPOINT=$TEI_EMBEDDING_ENDPOINT -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e REDIS_URL=$REDIS_URL -e INDEX_NAME=$INDEX_NAME opea/retriever-redis:comps + + sleep 3m +} + +function validate_microservice() { + retriever_port=5009 + export PATH="${HOME}/miniforge3/bin:$PATH" + source activate + sleep 1s + test_embedding=$(python -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)") + sleep 1s + http_proxy='' curl http://${ip_address}:$retriever_port/v1/retrieval \ + -X POST \ + -d '{"text":"test","embedding":${test_embedding}}' \ + -H 'Content-Type: application/json' +} + +function stop_docker() { + cid=$(docker ps -aq --filter "name=test-comps-retrievers*") + if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi +} + +function main() { + + stop_docker + + build_docker_images + start_service + + validate_microservice + + stop_docker + echo y | docker system prune + +} + +main