Merge pull request #5 from MagicAPI/perf_improvements

Use Thread and various performance improvements
MagicAPI · Nov 13, 2024 · 6164acf · 6164acf
2 parents f54584a + e2477cc
commit 6164acf
Show file tree

Hide file tree

Showing 12 changed files with 575 additions and 157 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -0,0 +1,10 @@
+target/
+.git/
+.github/
+.env
+**/*.rs.bk
+Dockerfile
+.dockerignore
+.gitignore
+README.md
+*.log 
diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
@@ -0,0 +1,105 @@
+name: Docker Build and Push
+
+on:
+  push:
+    branches:
+      - 'main'  # Only trigger push events on main
+    tags: [ "v*" ]
+  pull_request:  # Keep PR triggers for all branches
+  workflow_dispatch:
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  docker:
+    # Skip this job if it's a push event on a non-main branch
+    if: github.event_name != 'push' || github.ref == 'refs/heads/main'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      # Generate Cargo.lock if it doesn't exist
+      - name: Generate Cargo.lock
+        run: |
+          if [ ! -f "Cargo.lock" ]; then
+            cargo generate-lockfile
+          fi
+
+      - name: Extract version from Cargo.toml
+        id: version
+        run: |
+          VERSION=$(awk -F '"' '/^version = / {print $2}' Cargo.toml)
+          echo "CARGO_VERSION=${VERSION}" >> $GITHUB_ENV
+          echo "Version found: ${VERSION}"
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      # Always build to verify Dockerfile, but push only on main
+      - name: Build Docker image (non-main branch)
+        if: github.ref != 'refs/heads/main'
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: false
+          platforms: linux/amd64,linux/arm64
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      # Main branch handling - build and push
+      - name: Log in to GitHub Container Registry
+        if: github.ref == 'refs/heads/main'
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata for GHCR
+        if: github.ref == 'refs/heads/main'
+        id: meta-ghcr
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=raw,value=latest
+            type=raw,value=${{ env.CARGO_VERSION }}
+
+      - name: Build and push to GitHub Container Registry
+        if: github.ref == 'refs/heads/main'
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta-ghcr.outputs.tags }}
+          labels: ${{ steps.meta-ghcr.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          platforms: linux/amd64,linux/arm64
+
+      - name: Login to Docker Hub
+        if: github.ref == 'refs/heads/main'
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push to Docker Hub
+        if: github.ref == 'refs/heads/main'
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          tags: |
+            magicapi1/magicapi-ai-gateway:latest
+            magicapi1/magicapi-ai-gateway:${{ env.CARGO_VERSION }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          platforms: linux/amd64
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.1.7] - 2024-11-13
+### Added
+- Managed deployment offering with testing gateway at gateway.magicapi.dev
+- Thread-based performance optimizations for improved request handling
+- Documentation for testing deployment environment
+### Enhanced
+- Significant performance improvements in request processing
+- Build system optimizations
+- CI/CD pipeline improvements
+### Fixed
+- Git build configuration issues
+- Various minor bug fixes
+
 ## [0.1.6] - 2024-11-13
 ### Added
 - Support for Fireworks AI provider
@@ -65,7 +78,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Error handling
 - Basic documentation
 
-[Unreleased]: https://github.com/MagicAPI/ai-gateway/compare/v0.1.6...HEAD
+[Unreleased]: https://github.com/MagicAPI/ai-gateway/compare/v0.1.7...HEAD
+[0.1.7]: https://github.com/MagicAPI/ai-gateway/compare/v0.1.6...v0.1.7
 [0.1.6]: https://github.com/MagicAPI/ai-gateway/compare/v0.1.5...v0.1.6
 [0.1.5]: https://github.com/MagicAPI/ai-gateway/compare/v0.1.4...v0.1.5
 [0.1.3]: https://github.com/MagicAPI/ai-gateway/compare/v0.1.0...v0.1.3
diff --git a/Cargo.toml b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "magicapi-ai-gateway"
-version = "0.1.6"
+version = "0.1.7"
 edition = "2021"
 description = "A high-performance AI Gateway proxy for routing requests to various AI providers, offering seamless integration and management of multiple AI services"
 authors = ["MagicAPI Team <team@magicapi.com>"]
@@ -12,6 +12,7 @@ readme = "README.md"
 keywords = ["ai", "gateway", "proxy", "openai", "llm"]
 categories = ["web-programming", "api-bindings", "asynchronous"]
 exclude = [
+    ".env",
     ".cursorrules",
     ".github/**/*",
     ".cargo_vcs_info.json",
@@ -20,22 +21,28 @@ exclude = [
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
+[profile.release]
+opt-level = 3
+lto = "fat"
+codegen-units = 1
+panic = "abort"
+strip = true
+debug = false
+
 [dependencies]
-axum = { version = "0.7", features = ["http2"] }
-tokio = { version = "1.0", features = ["full"] }
-tower-http = { version = "0.5", features = ["cors"] }
-tracing = "0.1"
-tracing-subscriber = { version = "0.3", features = ["env-filter"] }
-reqwest = { version = "0.11", features = ["stream", "json"] }
-futures = "0.3"
+axum = { version = "0.7", features = ["http2", "tokio"] }
+tokio = { version = "1.0", features = ["full", "parking_lot", "rt-multi-thread"] }
+tower-http = { version = "0.5", features = ["cors", "compression-full"] }
+tracing = { version = "0.1", features = ["attributes"] }
+tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }
+reqwest = { version = "0.12.9", features = ["stream", "json", "rustls-tls", "http2", "gzip", "brotli"], default-features = false }
 http = "1.0"
-tower = "0.4"
-bytes = "1.0"
+bytes = { version = "1.0", features = ["serde"] }
 dotenv = "0.15"
-futures-util = "0.3"
+futures-util = { version = "0.3", features = ["io"] }
 once_cell = "1.18"
-hyper = { version = "1.0", features = ["full"] }
 async-trait = "0.1"
 thiserror = "1.0"
 serde = { version = "1.0", features = ["derive"] }
-serde_json = "1.0"
+serde_json = "1.0"
+num_cpus = "1.15"
diff --git a/Dockerfile b/Dockerfile
@@ -1,5 +1,5 @@
 # Build stage
-FROM rust:1.82-slim-bookworm as builder
+FROM --platform=linux/amd64 rust:1.82-slim-bookworm as builder
 
 # Install required dependencies
 RUN apt-get update && apt-get install -y \
@@ -9,22 +9,38 @@ RUN apt-get update && apt-get install -y \
 
 # Create a new empty shell project
 WORKDIR /usr/src/app
-COPY . .
 
-# Build with release profile for maximum performance
-RUN cargo build --release
+# Copy only necessary files first
+COPY Cargo.toml Cargo.lock ./
+
+# Create a dummy main.rs to build dependencies
+RUN mkdir src && \
+    echo "fn main() {}" > src/main.rs && \
+    cargo build --release --target x86_64-unknown-linux-gnu && \
+    rm -rf src
+
+# Now copy the real source code
+COPY src ./src
+
+# Build the application
+RUN RUSTFLAGS='-C target-feature=+crt-static' cargo build --release --target x86_64-unknown-linux-gnu && \
+    strip target/x86_64-unknown-linux-gnu/release/magicapi-ai-gateway
 
 # Runtime stage
-FROM debian:bookworm-slim
+FROM --platform=linux/amd64 debian:bookworm-slim
+
+# Add LABEL to identify the image
+LABEL org.opencontainers.image.source="https://github.com/magicapi/ai-gateway"
+LABEL org.opencontainers.image.description="MagicAPI AI Gateway"
+LABEL org.opencontainers.image.version="latest"
 
 # Install runtime dependencies
 RUN apt-get update && apt-get install -y \
     ca-certificates \
-    libssl3 \
     && rm -rf /var/lib/apt/lists/*
 
 # Copy the binary from builder
-COPY --from=builder /usr/src/app/target/release/magicapi-ai-gateway /usr/local/bin/
+COPY --from=builder /usr/src/app/target/x86_64-unknown-linux-gnu/release/magicapi-ai-gateway /usr/local/bin/
 
 # Set the startup command
 CMD ["magicapi-ai-gateway"] 
diff --git a/README.md b/README.md
@@ -249,10 +249,15 @@ Special thanks to all our contributors and the Rust community for making this pr
 
 1. Build the Docker image:
 ```bash
-docker build -t magicapi1/magicapi-ai-gateway:latest .
+docker buildx build --platform linux/amd64 -t magicapi1/magicapi-ai-gateway:latest . --load
 ```
 
-2. Run the container:
+2. Push the image to Docker Hub:
+```bash
+docker push magicapi1/magicapi-ai-gateway:latest
+```
+
+3. Run the container:
 ```bash
 docker run -p 3000:3000 \
   -e RUST_LOG=info \
@@ -279,6 +284,7 @@ version: '3.8'
 services:
   gateway:
     build: .
+    platform: linux/amd64
     ports:
       - "3000:3000"
     environment:
@@ -295,6 +301,7 @@ version: '3.8'
 services:
   gateway:
     image: magicapi1/magicapi-ai-gateway:latest
+    platform: linux/amd64
     ports:
       - "3000:3000"
     environment:
@@ -327,11 +334,11 @@ git add Cargo.toml CHANGELOG.md
 git commit -m "chore: release v0.1.6"
 
 # Create a git tag
-git tag -a v0.1.7 -m "Release v0.1.6"
+git tag -a v0.1.7 -m "Release v0.1.7"
 
 # Push changes and tag
-git push origin release/v0.1.6
-git push origin v0.1.6
+git push origin release/v0.1.7
+git push origin v0.1.7
 ```
 
 ### 3. Publishing to crates.io
@@ -362,12 +369,42 @@ After publishing, verify:
 - The new version appears on [crates.io](https://crates.io/crates/magicapi-ai-gateway)
 - Documentation is updated on [docs.rs](https://docs.rs/magicapi-ai-gateway)
 - The GitHub release is visible (if using GitHub)
-```
+
 
 This process follows Rust community best practices for releasing crates. Remember to:
 - Follow semantic versioning (MAJOR.MINOR.PATCH)
 - Test thoroughly before releasing
 - Document all significant changes
 - Keep your repository and crates.io package in sync
 
-Would you like me to explain any part of this process in more detail?
+Would you like me to explain any part of this process in more detail?
+
+## Testing Deployment
+
+MagicAPI provides a testing deployment of the AI Gateway, hosted in our London data centre. This deployment is intended for testing and evaluation purposes only, and should not be used for production workloads.
+
+### Testing Gateway URL
+```
+https://gateway.magicapi.dev
+```
+
+### Example Request to Testing Gateway
+```bash
+curl --location 'https://gateway.magicapi.dev/v1/chat/completions' \
+  --header 'Authorization: Bearer YOUR_API_KEY' \
+  --header 'Content-Type: application/json' \
+  --header 'x-provider: groq' \
+  --data '{
+    "model": "llama-3.1-8b-instant",
+    "messages": [
+        {
+            "role": "user",
+            "content": "Write a poem"
+        }
+    ],
+    "stream": true,
+    "max_tokens": 300
+}'
+```
+
+> **Note**: This deployment is provided for testing and evaluation purposes only. For production workloads, please deploy your own instance of the gateway or contact us for information about production-ready managed solutions.