Build and deploy demo #1509
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and deploy demo | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '0 0 * * *' | |
| jobs: | |
| scheduled: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| name: Check out repo | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.12" | |
| cache: pip | |
| cache-dependency-path: setup.py | |
| - name: Install Python dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install sqlite-utils>=3.1 | |
| sqlite-utils --version | |
| pip install -e . | |
| pip install datasette | |
| pip install bs4 | |
| - name: Create auth.json | |
| env: | |
| GITHUB_ACCESS_TOKEN: ${{ secrets.GH_TOKEN }} | |
| run: | | |
| echo "{\"github_personal_token\": \"$GITHUB_ACCESS_TOKEN\"}" > auth.json | |
| - name: Fetch previous copy of database | |
| if: |- | |
| !contains(github.event.head_commit.message, 'REFRESH_DB') | |
| run: |- | |
| # So we can keep track of when we first saw each dependent repo | |
| wget https://github-to-sqlite.dogsheep.net/github.db | |
| - name: Fetch the repos | |
| run: |- | |
| github-to-sqlite repos github.db dogsheep | |
| sqlite-utils tables --counts github.db | |
| - name: Fetch releases, commits, issues, contributors | |
| run: |- | |
| sqlite-utils github.db "select full_name from repos where owner = 53015001 union select 'simonw/datasette' as full_name union select 'simonw/sqlite-utils' as full_name" \ | |
| --csv --no-headers | while read repo; | |
| do github-to-sqlite releases \ | |
| github.db $(echo $repo | tr -d '\r'); | |
| sleep 10; | |
| github-to-sqlite commits \ | |
| github.db $(echo $repo | tr -d '\r'); | |
| sleep 10; | |
| github-to-sqlite tags \ | |
| github.db $(echo $repo | tr -d '\r'); | |
| sleep 10; | |
| github-to-sqlite contributors \ | |
| github.db $(echo $repo | tr -d '\r'); | |
| sleep 10; | |
| github-to-sqlite issues \ | |
| github.db $(echo $repo | tr -d '\r'); | |
| sleep 10; | |
| github-to-sqlite pull-requests \ | |
| github.db $(echo $repo | tr -d '\r'); | |
| sleep 10; | |
| github-to-sqlite issue-comments \ | |
| github.db $(echo $repo | tr -d '\r'); | |
| sleep 10; | |
| github-to-sqlite stargazers \ | |
| github.db $(echo $repo | tr -d '\r'); | |
| sleep 10; | |
| github-to-sqlite workflows \ | |
| github.db $(echo $repo | tr -d '\r'); | |
| sleep 10; | |
| done; | |
| # Scrape dependents | |
| github-to-sqlite scrape-dependents github.db simonw/datasette simonw/sqlite-utils -v | |
| sqlite-utils tables --counts github.db | |
| # Delete email addresses from raw_authors | |
| sqlite-utils github.db "update raw_authors set email = ''" | |
| # Fetch emojis | |
| github-to-sqlite emojis github.db --fetch | |
| # Rebuild FTS tables | |
| sqlite-utils rebuild-fts github.db | |
| # Populate _analyze_tables_ table | |
| sqlite-utils analyze-tables github.db --save | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| path: github.db | |
| - name: Set up Cloud Run | |
| uses: google-github-actions/setup-gcloud@v0 | |
| with: | |
| version: '318.0.0' | |
| service_account_email: ${{ secrets.GCP_SA_EMAIL }} | |
| service_account_key: ${{ secrets.GCP_SA_KEY }} | |
| - name: Deploy to Cloud Run | |
| run: |- | |
| gcloud config set run/region us-central1 | |
| gcloud config set project datasette-222320 | |
| datasette publish cloudrun github.db \ | |
| -m demo-metadata.json \ | |
| --service github-to-sqlite \ | |
| --install=datasette-search-all>=0.3 \ | |
| --install=datasette-render-markdown>=1.1.2 \ | |
| --install=datasette-pretty-json>=0.2.2 \ | |
| --install=datasette-json-html \ | |
| --install=datasette-vega \ | |
| --install=datasette-render-images \ | |
| --install=datasette-graphql \ | |
| --install=datasette-atom |