Test #451
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test | |
| on: | |
| pull_request: | |
| schedule: | |
| - cron: "0 13 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| openai_model: | |
| description: 'Model name for OpenAI tests' | |
| type: string | |
| required: false | |
| openai_vision_model: | |
| description: 'Model name for OpenAI vision tests' | |
| type: string | |
| required: false | |
| anthropic_model: | |
| description: 'Model name for Anthropic tests' | |
| type: string | |
| required: false | |
| google_model: | |
| description: 'Model name for Google tests' | |
| type: string | |
| required: false | |
| mistralai_model: | |
| description: 'Model name for Mistralai tests' | |
| type: string | |
| required: false | |
| lamini_model: | |
| description: 'Model name for Lamini tests' | |
| type: string | |
| required: false | |
| magentic_model: | |
| description: 'Model name for Magentic tests' | |
| type: string | |
| required: false | |
| env: | |
| PYTHON_VERSION: 3.11.4 | |
| jobs: | |
| test: | |
| runs-on: ubuntu-latest | |
| env: | |
| LOG10_URL: "https://log10.io" | |
| LOG10_ORG_ID: ${{ secrets.LOG10_ORG_ID }} | |
| LOG10_TOKEN: ${{ secrets.LOG10_TOKEN }} | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} | |
| LAMINI_API_KEY: ${{ secrets.LAMINI_API_KEY }} | |
| GOOGLE_API_KEY : ${{ secrets.GOOGLE_API_KEY }} | |
| PERPLEXITYAI_API_KEY: ${{ secrets.PERPLEXITYAI_API_KEY }} | |
| CEREBRAS_API_KEY: ${{ secrets.CEREBRAS_API_KEY }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install poetry | |
| run: pipx install poetry | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{env.PYTHON_VERSION}} | |
| cache: "poetry" | |
| architecture: 'x64' | |
| - name: Install dependencies | |
| run: poetry install --all-extras | |
| - name: Run cli tests | |
| run: poetry run pytest -vv tests/test_cli.py | |
| - name: Run dispatch llm tests | |
| if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request' }} | |
| run: | | |
| echo "This is a dispatch event" | |
| openai_model_input=${{ github.event.inputs.openai_model }} | |
| openai_vision_model_input=${{ github.event.inputs.openai_vision_model }} | |
| anthropic_model_input=${{ github.event.inputs.anthropic_model }} | |
| google_model_input=${{ github.event.inputs.google_model }} | |
| mistralai_model_input=${{ github.event.inputs.mistralai_model }} | |
| lamini_model_input=${{ github.event.inputs.lamini_model }} | |
| magentic_model_input=${{ github.event.inputs.magentic_model }} | |
| empty_inputs=true | |
| if [[ -n "$openai_model_input" ]]; then | |
| empty_inputs=false | |
| poetry run pytest --openai_model=$openai_model_input -vv tests/test_openai.py | |
| fi | |
| if [[ -n "$openai_vision_model_input" ]]; then | |
| empty_inputs=false | |
| poetry run pytest --openai_vision_model=$openai_vision_model_input -m vision -vv tests/test_openai.py | |
| fi | |
| if [[ -n "$anthropic_model_input" ]]; then | |
| empty_inputs=false | |
| poetry run pytest --anthropic_model=$anthropic_model_input -vv tests/test_anthropic.py | |
| fi | |
| if [[ -n "$google_model_input" ]]; then | |
| empty_inputs=false | |
| poetry run pytest --google_model=$google_model_input -vv tests/test_google.py | |
| fi | |
| if [[ -n "$mistralai_model_input" ]]; then | |
| empty_inputs=false | |
| poetry run pytest --mistralai_model=$mistralai_model_input -vv tests/test_mistralai.py | |
| fi | |
| if [[ -n "$lamini_model_input" ]]; then | |
| empty_inputs=false | |
| poetry run pytest --lamini_model=$lamini_model_input -vv tests/test_lamini.py | |
| fi | |
| if [[ -n "$llm_provider_input" ]]; then | |
| empty_inputs=false | |
| poetry run pytest --llm_provider=$llm_provider_input -vv tests/test_magentic.py | |
| fi | |
| if $empty_inputs; then | |
| echo "All variables are empty" | |
| poetry run pytest -vv tests/ --ignore=tests/test_cli.py | |
| poetry run pytest --llm_provider=anthropic -vv tests/test_magentic.py | |
| poetry run pytest --llm_provider=litellm --openai_compatibility_model=perplexity/llama-3.1-sonar-small-128k-chat -vv tests/test_magentic.py -m chat | |
| fi | |
| - name: Run scheduled llm tests | |
| if: ${{ github.event_name == 'schedule' }} | |
| run: | | |
| echo "This is a schedule event" | |
| poetry run pytest -vv tests/ --ignore=tests/test_cli.py | |
| poetry run pytest --openai_model=gpt-4o -m chat -vv tests/test_openai.py | |
| poetry run pytest --llm_provider=litellm --openai_compatibility_model=perplexity/llama-3.1-sonar-small-128k-chat -vv tests/test_magentic.py -m chat |