test_openai.yml

name: OpenAI GPT-4 Capabilities Test

env:
  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
  COMPOSIO_API_KEY: ${{ secrets.COMPOSIO_API_KEY }}

on:
  push:
    branches: [ main ]
  pull_request:
    branches: [ main ]

jobs:
  test:
    runs-on: ubuntu-latest
    timeout-minutes: 15
    steps:
    - name: Checkout
      uses: actions/checkout@v4

    - name: "Setup Python, Poetry and Dependencies"
      uses: packetcoders/action-setup-cache-python-poetry@main
      with:
        python-version: "3.12"
        poetry-version: "1.8.2"
        install-args: "-E dev -E external-tools"

    - name: Test first message contains expected function call and inner monologue
      id: test_first_message
      env:
        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      run: |
        poetry run pytest -s -vv tests/test_model_letta_perfomance.py::test_openai_gpt_4o_returns_valid_first_message

    - name: Test model sends message with keyword
      id: test_keyword_message
      env:
        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      run: |
        poetry run pytest -s -vv tests/test_model_letta_perfomance.py::test_openai_gpt_4o_returns_keyword

    - name: Test model uses external tool correctly
      id: test_external_tool
      env:
        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      run: |
        poetry run pytest -s -vv tests/test_model_letta_perfomance.py::test_openai_gpt_4o_uses_external_tool

    - name: Test model recalls chat memory
      id: test_chat_memory
      env:
        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      run: |
        poetry run pytest -s -vv tests/test_model_letta_perfomance.py::test_openai_gpt_4o_recall_chat_memory

    - name: Test model uses 'archival_memory_search' to find secret
      id: test_archival_memory_search
      env:
        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      run: |
        poetry run pytest -s -vv tests/test_model_letta_perfomance.py::test_openai_gpt_4o_archival_memory_retrieval

    - name: Test model uses 'archival_memory_insert' to insert archival memories
      id: test_archival_memory_insert
      env:
        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      run: |
        poetry run pytest -s -vv tests/test_model_letta_perfomance.py::test_openai_gpt_4o_archival_memory_insert

    - name: Test model can edit core memories
      id: test_core_memory
      env:
        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      run: |
        poetry run pytest -s -vv tests/test_model_letta_perfomance.py::test_openai_gpt_4o_edit_core_memory

    - name: Test embedding endpoint
      id: test_embedding_endpoint
      env:
        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      run: |
        poetry run pytest -s -vv tests/test_model_letta_perfomance.py::test_embedding_endpoint_openai