tool benchmarking (#190)

Co-authored-by: Bagatur <baskaryan@gmail.com>
langchain-ai · Jul 24, 2024 · 7e16b6d · 7e16b6d
1 parent 22d279a
commit 7e16b6d
Show file tree

Hide file tree

Showing 7 changed files with 2,627 additions and 1 deletion.
diff --git a/.github/workflows/tool_benchmarks.yml b/.github/workflows/tool_benchmarks.yml
@@ -0,0 +1,33 @@
+name: Weekly Tool Benchmarks
+
+on:
+ workflow_dispatch:
+ schedule:
+ - cron: '0 0 * * 0' # Runs at midnight (00:00) every Sunday (UTC time)
+
+jobs:
+ run_tool_benchmarks:
+ runs-on: ubuntu-latest
+
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Set up Python 3.12 + Poetry ${{ env.POETRY_VERSION }}
+ uses: "./.github/actions/poetry_setup"
+ with:
+ python-version: '3.12'
+ poetry-version: ${{ env.POETRY_VERSION }}
+ working-directory: .
+ cache-key: benchmarks-all
+
+ - name: Install dependencies
+ shell: bash
+ run: |
+ echo "Running tests, installing dependencies with poetry..."
+ poetry install --with test,lint,typing,docs
+
+ - name: Multiverse math benchmark
+ run: python scripts/multiverse_math_benchmark.py
+
+ - name: Query analysis benchmark
+ run: python scripts/query_analysis_benchmark.py
diff --git a/docs/source/notebooks/tool_usage/multiverse_math_benchmark.ipynb b/docs/source/notebooks/tool_usage/multiverse_math_benchmark.ipynb