chore: setup ci

.github/workflows/build-and-deploy.yml

@@ -0,0 +1,82 @@
+on:
+  push:
+  pull_request:
+  repository_dispatch:
+    types: build-and-deploy
+  workflow_dispatch:
+  workflow_call:
+
+concurrency:
+  group: cli-${{ github.workflow }}-${{ github.ref_type }}-${{ github.event.pull_request.number || github.ref || github.run_id }}
+  cancel-in-progress: true
+
+defaults:
+  run:
+    shell: bash -euxo pipefail {0}
+
+env:
+  GITHUB_REPOSITORY_URL: ${{ github.server_url }}/${{ github.repository }}
+  VERBOSE: 1
+
+jobs:
+  build-and-deploy-datasets:
+    runs-on: ubuntu-22.04
+
+    environment:
+      name: ${{ github.ref }}
+
+    env:
+      VERBOSE: 0
+      PYTHONUNBUFFERED: 1
+      DATA_AWS_S3_BUCKET: ${{ secrets.DATA_AWS_S3_BUCKET }}
+      DATA_AWS_CLOUDFRONT_DISTRIBUTION_ID: ${{ secrets.DATA_AWS_CLOUDFRONT_DISTRIBUTION_ID }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_DEFAULT_REGION: us-east-2
+
+    steps:
+      - name: "Checkout code"
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 50
+          submodules: true
+
+      - name: "Install system dependencies"
+        run: |
+          sudo apt-get install brotli pigz parallel rename --yes -qq >/dev/null
+
+      - name: "Install awscli"
+        run: |
+          pushd /tmp >/dev/null
+            curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+            unzip -oqq awscliv2.zip
+            sudo ./aws/install --update
+          popd >/dev/null
+          aws --version
+
+      - name: "Authenticate git"
+        run: |
+          export GITHUB_TOKEN="${{ secrets.GH_TOKEN_NEXTSTRAIN_BOT_REPO }}"
+          gh auth setup-git >/dev/null
+
+      - name: "Rebuild, commit and push datasets"
+        if: github.ref != 'refs/heads/release'
+        run: |
+          ./scripts/rebuild_v3 --input-dir 'data_v3' --output-dir 'data_v3_output' --push --repo="${GITHUB_REPOSITORY}"
+
+      - name: "Rebuild, commit, push and deploy datasets"
+        if: github.ref == 'refs/heads/release'
+        run: |
+          ./scripts/rebuild_v3 --input-dir 'data_v3' --output-dir 'data_v3_output' --release --repo="${GITHUB_REPOSITORY}"
+
+      - name: "Upload build artifacts: dataset server"
+        uses: actions/upload-artifact@v3
+        with:
+          name: server
+          path: ./data_v3_output/*
+
+      - name: "Upload build artifacts: zip archives"
+        uses: actions/upload-artifact@v3
+        with:
+          name: zips
+          path: ./data_v3_temp/*

scripts/rebuild_v3

@@ -16,7 +16,7 @@ from lib.container import dict_get, dict_get_required, dict_set, unique, find_in
 from lib.date import now_iso, iso_to_iso_safe
 from lib.fs import json_read, find_files, json_write, copy, make_zip
 from lib.git import git_get_modified_files, git_dir_is_clean, git_get_dirty_files, git_check_tag, \
-  git_get_initial_commit_hash, github_create_release, git_pull, git_commit_and_push
+  git_get_initial_commit_hash, github_create_release, git_pull, git_commit_all, git_push
 
 logging.basicConfig(level=logging.INFO)
 l = logging.getLogger(__file__)
@@ -144,6 +144,12 @@ def parse_args():
   if args.release and not args.repo:
     parser.error("--release requires --repo")
 
+  if args.release:
+    args.push = True
+
+  if args.push:
+    args.commit = True
+
   return args
 
 
@@ -185,11 +191,14 @@ def main():
   json_write({**index_json, "updatedAt": updated_at}, index_json_path, no_sort_keys=True)
 
   if args.commit:
-    commit_hash = commit_changes(tag, release_infos)
+    commit_hash = commit_changes(args, tag, release_infos)
 
-    if args.release:
-      release_notes = aggregate_release_notes(release_infos)
-      publish_to_github_releases(args, tag, commit_hash, release_notes)
+    if args.push:
+      git_push()
+
+      if args.release:
+        release_notes = aggregate_release_notes(release_infos)
+        publish_to_github_releases(args, tag, commit_hash, release_notes)
 
 
 def process_one_collection(collection_json_path, args, tag, updated_at):
@@ -275,13 +284,18 @@ def aggregate_release_notes(release_infos):
   return release_notes
 
 
-def commit_changes(tag, release_infos):
+def commit_changes(args, tag, release_infos):
   l.info(f"Commiting changes for '{tag}'")
-  dataset_names = format_list(
-    unique([get_dataset_name(release_info["dataset"]) for release_info in release_infos]),
-    sep="\n", marker="- ", quote=False
-  )
-  return git_commit_and_push(commit_message=f"chore: release '{tag}'\n\nUpdated datasets:\n\n{dataset_names}")
+
+  commit_message = "chore: rebuild"
+  if args.release:
+    dataset_names = format_list(
+      unique([get_dataset_name(release_info["dataset"]) for release_info in release_infos]),
+      sep="\n", marker="- ", quote=False
+    )
+    commit_message = f"chore: release '{tag}'\n\nUpdated datasets:\n\n{dataset_names}"
+
+  return git_commit_all(commit_message)
 
 
 def publish_to_github_releases(args, tag, commit_hash, release_notes):