FULL_ETL #10
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: FULL_ETL | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '0 0 * * 0' | |
| env: | |
| SUPABASE_API_KEY: ${{ secrets.SUPABASE_API_KEY }} | |
| SUPABASE_API_URL: ${{ secrets.SUPABASE_API_URL }} | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| SUPABASE_DB_PASSWORD: ${{ secrets.PRODUCTION_DB_PASSWORD }} | |
| SUPABASE_DB_USER: ${{ secrets.PROD_DB_USER }} | |
| SUPABASE_DB_HOST: ${{ secrets.PROD_DB_HOST }} | |
| SUPABASE_DB_PORT: ${{ secrets.PROD_DB_PORT }} | |
| jobs: | |
| rss-etl: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout Repo Content | |
| uses: actions/checkout@v3 | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| - name: Setup Python Path | |
| run: echo "PYTHONPATH=$PYTHONPATH:." >> $GITHUB_ENV | |
| - name: Install Python Packages | |
| run: | | |
| python -m pip install python-docx | |
| python -m pip install supabase | |
| python -m pip install python-dotenv | |
| python -m pip install feedparser | |
| python -m pip install psycopg2 | |
| - name: Run RSS extraction script | |
| run: python law_reader/rss_etl.py | |
| docx-etl: | |
| needs: rss-etl | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout Repo Content | |
| uses: actions/checkout@v3 | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| - name: Setup Python Path | |
| run: echo "PYTHONPATH=$PYTHONPATH:." >> $GITHUB_ENV | |
| - name: Install Python Packages | |
| run: | | |
| python -m pip install python-docx | |
| python -m pip install supabase | |
| python -m pip install python-dotenv | |
| python -m pip install requests | |
| python -m pip install psycopg2 | |
| python -m pip install feedparser | |
| - name: Install Libreoffice for doc to docx conversion | |
| run: | | |
| sudo apt-get update | |
| sudo apt install libreoffice | |
| - name: Run DOCX Text extraction script | |
| run: python law_reader/docx_etl.py | |
| summarize_etl: | |
| needs: docx-etl | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout Repo Content | |
| uses: actions/checkout@v3 | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| - name: Install Python Packages | |
| run: | | |
| python -m pip install supabase | |
| python -m pip install python-dotenv | |
| python -m pip install openai | |
| python -m pip install langchain | |
| python -m pip install gpt4all | |
| python -m pip install chromadb | |
| python -m pip install feedparser | |
| python -m pip install psycopg2 | |
| # Add other packages as needed | |
| - name: Run Summarize ETL script | |
| run: | | |
| PYTHONPATH=$(pwd) python law_reader/summarize_etl.py | |