Save audio files to gcs (#2)

* add dockerfile and deploy.yml

* remove groq_api_key from env_var

* add footer; improve readme, page title and subheader

* render a default state for sidebar content

* cleanup

* allow traffic on first creation

* reset no_traffic to true for preview urls

* remove the footer file

* rename service to audiora

* revert no_traffic

* set concurrency to 80

* set max-instances to 10

* add .env.example; cleanup

* create a storage service; define bucket_name env variable

* use a call abstraction for gcs methods

* add logic to store an audio to gcs

* fix bug with audio enhancement logic; cleanup

.env.example

@@ -0,0 +1,10 @@
+PROJECT_ID="your-project-id"
+GOOGLE_APPLICATION_CREDENTIALS="path/to/your/serviceAccount.json"
+
+OPENAI_API_KEY="your-openai-api-key"
+ANTHROPIC_API_KEY="your-anthropic-api-key"
+GEMINI_API_KEY="your-gemini-api-key"
+ELEVENLABS_API_KEY="your-elevenlabs-api-key"
+
+BUCKET_NAME="your-bucket-name"
+APP_URL=http://localhost:8501

.github/workflows/deploy.yml

@@ -26,27 +26,21 @@ env:
   GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
   ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
   ELEVENLABS_API_KEY: ${{ secrets.ELEVENLABS_API_KEY }}
+  BUCKET_NAME: ${{ secrets.BUCKET_NAME }}
 
 jobs:
   prepare:
     runs-on: ubuntu-latest
     outputs:
       VERSION: ${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.number) || format('main-{0}', steps.prepare_env.outputs.SHORT_SHA) }}
-      MAIN_OR_TAGGED: ${{ fromJSON(env.MAIN) || (fromJSON(steps.prepare_env.outputs.TAGGED) && steps.prepare_env.outputs.TAG_BRANCH_NAME == 'main') }}
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - id: prepare_env
         run: |
           echo "TAGGED=${{ startsWith(github.ref, 'refs/tags/api') }}" >> $GITHUB_OUTPUT
-
-          SHORT_SHA=$(git rev-parse --short HEAD)
-          echo "SHORT_SHA=$SHORT_SHA" >> $GITHUB_OUTPUT
-
-          RAW=$(git branch -r --contains $SHORT_SHA)
-          TAG_BRANCH_NAME="${RAW##*/}"
-          echo "TAG_BRANCH_NAME=$TAG_BRANCH_NAME" >> $GITHUB_OUTPUT
+          echo "SHORT_SHA=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
 
   lint:
     runs-on: ubuntu-latest
@@ -103,20 +97,20 @@ jobs:
             GEMINI_API_KEY=${{ secrets.GEMINI_API_KEY }}
             ANTHROPIC_API_KEY=${{ secrets.ANTHROPIC_API_KEY }}
             ELEVENLABS_API_KEY=${{ secrets.ELEVENLABS_API_KEY }}
+            BUCKET_NAME=${{ secrets.BUCKET_NAME }}
 
           flags: "--allow-unauthenticated --memory=32Gi --cpu=8 --execution-environment=gen2 --concurrency=80 --max-instances=10"
 
-      - name: health-check
-        run: curl -f "${{ steps.deploy.outputs.url }}"
+      - run: curl -f "${{ steps.deploy.outputs.url }}"
       - uses: marocchino/sticky-pull-request-comment@v2
         with:
-          header: api
+          header: app
           message: |
             app: ${{ steps.deploy.outputs.url }} (${{ github.event.pull_request.head.sha }})
 
   promote:
     runs-on: ubuntu-latest
-    if: (needs.prepare.outputs.MAIN_OR_TAGGED == 'true')
+    if: (needs.prepare.outputs.MAIN == 'true')
     needs: [prepare, deploy, lint]
     timeout-minutes: 3
     steps:

.gitignore

@@ -8,3 +8,5 @@ __pycache__
 !.env.example
 
 reference_code
+
+keys/

.streamlit/config.toml

@@ -0,0 +1,2 @@
+[browser]
+    gatherUsageStats = false

app.py

@@ -12,15 +12,7 @@ async def main():
 
     st.title("🎧 Audiora")
     st.subheader("Listen to anything, anytime, leveraging AI")
-
-    st.sidebar.markdown(
-        """
-        <p> 
-            A <a href="https://veedo.ai">VeedoAI</a> project. (c) 2024 
-        </p>
-        """,
-        unsafe_allow_html=True,
-    )
+    st.sidebar.info("A VeedoAI project. (c) 2024")
 
     # Sidebar for content type selection
     st.sidebar.title("Audiocast Info")

src/env_var.py

@@ -12,4 +12,5 @@
 GEMINI_API_KEY = environ["GEMINI_API_KEY"]
 ELEVENLABS_API_KEY = environ["ELEVENLABS_API_KEY"]
 
+BUCKET_NAME = environ.get("BUCKET_NAME")
 APP_URL = environ.get("APP_URL", "http://localhost:8501")

src/services/storage.py

@@ -0,0 +1,77 @@
+from dataclasses import dataclass
+from io import BytesIO
+from pathlib import Path
+from typing import Any, Dict
+from uuid import uuid4
+
+from google.cloud import storage
+
+from src.env_var import BUCKET_NAME
+
+storage_client = storage.Client()
+bucket = storage_client.bucket(BUCKET_NAME)
+BLOB_BASE_URI = "audiora/assets"
+
+
+def listBlobs(prefix):
+    blobs = bucket.list_blobs(prefix=prefix)
+    return [blob for blob in blobs]
+
+
+def check_file_exists(root_path: str, filename: str):
+    """check if a file exists in the bucket"""
+    blobname = f"{root_path}/{filename}"
+    blobs = listBlobs(prefix=root_path)
+    return any(blob.name == blobname for blob in blobs)
+
+
+@dataclass
+class UploadItemParams:
+    content_type: str
+    cache_control: str = "public, max-age=31536000"
+    metadata: Dict[str, Any] | None = None
+
+
+class StorageManager:
+    def upload_to_gcs(
+        self, item: str | Path | BytesIO, blobname: str, params: UploadItemParams
+    ):
+        """upload item to GCS"""
+        blob = bucket.blob(blobname)
+        blob.content_type = params.content_type
+        blob.cache_control = params.cache_control
+
+        if params.metadata:
+            blob.metadata = {**(blob.metadata or dict()), **params.metadata}
+
+        if isinstance(item, Path):
+            blob.upload_from_filename(str(item))
+        elif isinstance(item, str):
+            blob.upload_from_string(item)
+        else:
+            blob.upload_from_file(item)
+
+        return f"gs://{BUCKET_NAME}/{blob.name}"
+
+    def upload_audio_to_gcs(self, tmp_audio_path: str, filename=str(uuid4())):
+        """upload audio file to GCS"""
+        blobname = f"{BLOB_BASE_URI}/{filename}"
+        self.upload_to_gcs(
+            Path(tmp_audio_path),
+            blobname,
+            UploadItemParams(content_type="audio/mpeg"),
+        )
+
+        return f"gs://{BUCKET_NAME}/{blobname}"
+
+    def download_from_gcs(self, filename: str):
+        """
+        Download any item on GCS to disk
+        """
+        blobname = f"{BLOB_BASE_URI}/{filename}"
+        blob = bucket.blob(blobname)
+        tmp_file_path = f"/tmp/{str(uuid4())}"
+
+        blob.download_to_filename(tmp_file_path)
+
+        return tmp_file_path

src/utils/audio_synthesizer.py

@@ -62,7 +62,9 @@ def enhance_audio_minimal(
             audio = audio.speedup(playback_speed=speed_factor)
             # Only normalize if significant volume differences
             if abs(audio.dBFS + 18.0) > 3.0:  # +/- 3dB tolerance
-                audio = audio.normalize(target_level=-18.0)
+                # Calculate gain needed to reach target dBFS of -18.0
+                gain_needed = -18.0 - audio.dBFS
+                audio = audio.apply_gain(gain_needed)
 
             # Export with high quality
             audio.export(

src/utils/main_utils.py

@@ -4,8 +4,8 @@
 
 import streamlit as st
 from pydantic import BaseModel
-from slugify import slugify
 
+from src.services.storage import StorageManager
 from src.utils.audio_manager import AudioManager
 from src.utils.audio_synthesizer import AudioSynthesizer
 from src.utils.audiocast_request import AudioScriptMaker, generate_source_content
@@ -24,7 +24,6 @@ class GenerateAudioCastRequest(BaseModel):
 
 class GenerateAudioCastResponse(BaseModel):
     uuid: str
-    slug: str
     url: str
     script: str
     source_content: str
@@ -89,32 +88,41 @@ async def generate_audiocast(request: GenerateAudioCastRequest):
         if not audio_script:
             raise Exception("Error while generating audio script")
 
-    # TODO: Ingest audio file to a storage service (e.g., GCS, S3) using a background service
     # STEP 3: Generate audio from the audio script
     with container.container():
         container.info("Generating audio...")
-        outputfile = await AudioManager().generate_speech(audio_script)
+        output_file = await AudioManager().generate_speech(audio_script)
 
         container.info("Enhancing audio quality...")
-        AudioSynthesizer().enhance_audio_minimal(Path(outputfile))
-        print(f"outputfile: {outputfile}")
+        AudioSynthesizer().enhance_audio_minimal(Path(output_file))
+        print(f"output_file: {output_file}")
 
-    # Generate slug from the query
-    slug = slugify((category + summary)[:50])  # First 50 chars for the slug
-    # Generate a unique ID for the audiocast
-    audiocast_id = str(uuid.uuid4())
+    # unique ID for the audiocast
+    uniq_id = str(uuid.uuid4())
+
+    # TODO: Use a background service
+    # STEP 4: Ingest audio file to a storage service (e.g., GCS, S3)
+    with container.container():
+        try:
+            container.info("Storing a copy of your audiocast...")
+            storage_manager = StorageManager()
+            storage_manager.upload_audio_to_gcs(output_file, uniq_id)
+        except Exception as e:
+            print(f"Error while storing audiocast: {str(e)}")
 
     response = GenerateAudioCastResponse(
-        uuid=audiocast_id,
-        slug=slug,
-        url=outputfile,
+        uuid=uniq_id,
+        url=output_file,
         script=audio_script,
         source_content=source_content,
     )
 
     return response.model_dump()
 
 
-async def get_audiocast(uuid: str):
-    # TODO: Implement audiocast retrieval
-    pass
+async def get_audiocast_uri(uuid: str):
+    """
+    Get the URI for the audiocast
+    """
+    storage_manager = StorageManager()
+    return storage_manager.download_from_gcs(uuid)

src/utils/render_audiocast.py

@@ -9,7 +9,6 @@
 
 class GenerateAudiocastDict(TypedDict):
     uuid: str
-    slug: str
     url: str
     script: str
     source_content: str
@@ -34,9 +33,7 @@ def render_audiocast():
     st.sidebar.subheader("Audiocast Source")
     st.sidebar.markdown(current_audiocast["source_content"])
 
-    share_url = (
-        f"{APP_URL}/audiocast/{current_audiocast['uuid']}/{current_audiocast['slug']}"
-    )
+    share_url = f"{APP_URL}/audiocast/{current_audiocast['uuid']}"
     st.text_input("Share this audiocast:", share_url)
 
     share_col, restart_row = st.columns(2, vertical_alignment="bottom")