Refactor dependency downloads: (#508)

- Checksum dependencies using taskfiles rather than a separate script.
- Move core's dependency download script into taskfiles.
- Add support for downloading yscope-log-viewer even when no .git directory exists.

.github/actions/clp-core-build/action.yaml

@@ -18,10 +18,6 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - shell: "bash"
-      working-directory: "./components/core"
-      run: "./tools/scripts/deps-download/download-all.sh"
-
     - if: "inputs.use_published_image == 'false'"
       uses: "actions/download-artifact@v4"
       with:
@@ -46,13 +42,19 @@ runs:
         fi
       shell: "bash"
 
+    - run: "./tools/scripts/deps-download/init.sh"
+      shell: "bash"
+
     - run: >-
         docker run
         --user $(id -u):$(id -g)
-        --volume "$GITHUB_WORKSPACE/components/core":/mnt/clp
+        --volume "$GITHUB_WORKSPACE":/mnt/clp
         --workdir /mnt/clp
         ${{steps.get_image_props.outputs.qualified_image_name}}
-        /mnt/clp/tools/scripts/utils/build-and-run-unit-tests.sh . build
+        bash -c "task deps:core &&
+        /mnt/clp/components/core/tools/scripts/utils/build-and-run-unit-tests.sh
+        /mnt/clp/components/core
+        /mnt/clp/components/core/build"
       shell: "bash"
 
     - if: "inputs.upload_binaries == 'true'"

.github/workflows/clp-core-build-macos.yaml

@@ -11,6 +11,9 @@ on:
       - "components/core/tools/scripts/lib_install/macos-12/**"
       - "components/core/tools/scripts/deps-download/**"
       - "components/core/tools/scripts/utils/build-and-run-unit-tests.sh"
+      - "deps-tasks.yml"
+      - "Taskfile.yml"
+      - "tools/scripts/deps-download/**"
   push:
     paths:
       - ".github/workflows/clp-core-build-macos.yaml"
@@ -21,6 +24,9 @@ on:
       - "components/core/tools/scripts/lib_install/macos-12/**"
       - "components/core/tools/scripts/deps-download/**"
       - "components/core/tools/scripts/utils/build-and-run-unit-tests.sh"
+      - "deps-tasks.yml"
+      - "Taskfile.yml"
+      - "tools/scripts/deps-download/**"
   workflow_dispatch:
 
 concurrency:
@@ -47,10 +53,11 @@ jobs:
       - name: "Install dependencies"
         run: "./components/core/tools/scripts/lib_install/macos-12/install-all.sh"
 
-      - name: "Download source dependencies"
+      - run: "./tools/scripts/deps-download/init.sh"
+        shell: "bash"
+
+      - run: "task deps:core"
         shell: "bash"
-        working-directory: "./components/core"
-        run: "./tools/scripts/deps-download/download-all.sh"
 
       - name: "Build CLP-core and run unit tests"
         shell: "bash"

.github/workflows/clp-core-build.yaml

@@ -8,6 +8,9 @@ on:
       - ".github/workflows/clp-core-build.yaml"
       - ".gitmodules"
       - "components/core/**"
+      - "deps-tasks.yml"
+      - "Taskfile.yml"
+      - "tools/scripts/deps-download/**"
       - "!components/core/tools/scripts/lib_install/macos-12/**"
   push:
     paths:
@@ -16,6 +19,9 @@ on:
       - ".github/workflows/clp-core-build.yaml"
       - ".gitmodules"
       - "components/core/**"
+      - "deps-tasks.yml"
+      - "Taskfile.yml"
+      - "tools/scripts/deps-download/**"
       - "!components/core/tools/scripts/lib_install/macos-12/**"
   workflow_dispatch:
 
@@ -83,8 +89,10 @@ jobs:
               - "components/core/CMakeLists.txt"
               - "components/core/src/**"
               - "components/core/tests/**"
-              - "components/core/tools/scripts/deps-download/**"
               - "components/core/tools/scripts/utils/build-and-run-unit-tests.sh"
+              - "deps-tasks.yml"
+              - "Taskfile.yml"
+              - "tools/scripts/deps-download/**"
 
   centos74-deps-image:
     if: "needs.filter-relevant-changes.outputs.centos74_image_changed == 'true'"

Taskfile.yml

@@ -1,16 +1,22 @@
 version: "3"
 
 includes:
+  deps: "deps-tasks.yml"
   docs: "docs/tasks.yml"
   lint: "lint-tasks.yml"
   utils: "tools/yscope-dev-utils/taskfiles/utils.yml"
 
 vars:
-  # Paths
+  # Source paths
+  G_COMPONENTS_DIR: "{{.ROOT_DIR}}/components"
+  G_CORE_COMPONENT_DIR: "{{.G_COMPONENTS_DIR}}/core"
+  G_CORE_COMPONENT_SUBMODULES_DIR: "{{.G_CORE_COMPONENT_DIR}}/submodules"
+  G_LOG_VIEWER_WEBUI_SRC_DIR: "{{.G_COMPONENTS_DIR}}/log-viewer-webui"
+
+  # Build paths
   G_BUILD_DIR: "{{.ROOT_DIR}}/build"
   G_CORE_COMPONENT_BUILD_DIR: "{{.G_BUILD_DIR}}/core"
   G_LOG_VIEWER_WEBUI_BUILD_DIR: "{{.G_BUILD_DIR}}/log-viewer-webui"
-  G_LOG_VIEWER_WEBUI_SRC_DIR: "{{.ROOT_DIR}}/components/log-viewer-webui"
   G_METEOR_BUILD_DIR: "{{.G_BUILD_DIR}}/meteor"
   G_NODEJS_14_BUILD_DIR: "{{.G_BUILD_DIR}}/nodejs-14"
   G_NODEJS_14_BIN_DIR: "{{.G_NODEJS_14_BUILD_DIR}}/bin"
@@ -20,6 +26,9 @@ vars:
   G_PACKAGE_VENV_DIR: "{{.G_BUILD_DIR}}/package-venv"
   G_WEBUI_BUILD_DIR: "{{.G_BUILD_DIR}}/webui"
 
+  # Taskfile paths
+  G_UTILS_TASKFILE: "{{.ROOT_DIR}}/tools/yscope-dev-utils/taskfiles/utils.yml"
+
   # Versions
   G_PACKAGE_VERSION: "0.2.0-dev"
 
@@ -171,7 +180,7 @@ tasks:
     vars:
       SRC_DIR: "components/core"
     sources:
-      - "{{.G_BUILD_DIR}}/core-submodules.md5"
+      - "{{.G_DEPS_CORE_CHECKSUM_FILE}}"
       - "{{.SRC_DIR}}/cmake/**/*"
       - "{{.SRC_DIR}}/CMakeLists.txt"
       - "{{.SRC_DIR}}/src/**/*"
@@ -183,7 +192,7 @@ tasks:
       - "{{.G_CORE_COMPONENT_BUILD_DIR}}/clp"
       - "{{.G_CORE_COMPONENT_BUILD_DIR}}/clp-s"
       - "{{.G_CORE_COMPONENT_BUILD_DIR}}/reducer-server"
-    deps: ["core-submodules", "init"]
+    deps: ["deps:core", "init"]
     cmds:
       - "mkdir -p '{{.G_CORE_COMPONENT_BUILD_DIR}}'"
       - "cmake -S '{{.SRC_DIR}}' -B '{{.G_CORE_COMPONENT_BUILD_DIR}}'"
@@ -317,31 +326,6 @@ tasks:
           NODEJS_VERSION: "v14.21.3"
           OUTPUT_DIR: "{{.OUTPUT_DIR}}"
 
-  core-submodules:
-    internal: true
-    vars:
-      CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5"
-      OUTPUT_DIR: "submodules"
-    sources:
-      - "{{.TASKFILE}}"
-      - ".gitmodules"
-      - "tools/scripts/deps-download/**/*"
-    dir: "components/core"
-    generates: ["{{.CHECKSUM_FILE}}"]
-    deps:
-      - "init"
-      - task: "utils:validate-checksum"
-        vars:
-          CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
-          DATA_DIR: "{{.OUTPUT_DIR}}"
-    cmds:
-      - "tools/scripts/deps-download/download-all.sh"
-      # This command must be last
-      - task: "utils:compute-checksum"
-        vars:
-          DATA_DIR: "{{.OUTPUT_DIR}}"
-          OUTPUT_FILE: "{{.CHECKSUM_FILE}}"
-
   download-and-extract-tar:
     internal: true
     label: "{{.TASK}}-{{.TAR_NAME}}"
@@ -399,7 +383,7 @@ tasks:
       PACKAGE_OUTPUT_DIR: "{{.SRC_DIR}}/node_modules"
       SERVER_OUTPUT_DIR: "{{.SRC_DIR}}/server/node_modules"
     sources:
-      - "{{.G_BUILD_DIR}}/log-viewer-webui-submodules.md5"
+      - "{{.G_DEPS_LOG_VIEWER_CHECKSUM_FILE}}"
       - "{{.G_BUILD_DIR}}/nodejs-22.md5"
       - "{{.TASKFILE}}"
       - "client/package.json"
@@ -418,7 +402,7 @@ tasks:
       - "{{.PACKAGE_CHECKSUM_FILE}}"
       - "{{.SERVER_CHECKSUM_FILE}}"
     deps:
-      - "log-viewer-webui-submodules"
+      - "deps:log-viewer"
       - "nodejs-22"
       - task: "utils:validate-checksum"
         vars:
@@ -469,30 +453,6 @@ tasks:
         "{{.SERVER_CHECKSUM_FILE}}"
         > "{{.CHECKSUM_FILE}}"
 
-  log-viewer-webui-submodules:
-    internal: true
-    vars:
-      CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5"
-      OUTPUT_DIR: "yscope-log-viewer"
-    sources:
-      - "{{.TASKFILE}}"
-      - ".gitmodules"
-    dir: "components/log-viewer-webui"
-    generates: ["{{.CHECKSUM_FILE}}"]
-    deps:
-      - "init"
-      - task: "utils:validate-checksum"
-        vars:
-          CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
-          DATA_DIR: "{{.OUTPUT_DIR}}"
-    cmds:
-      - "git submodule update --init --recursive yscope-log-viewer"
-      # This command must be last
-      - task: "utils:compute-checksum"
-        vars:
-          DATA_DIR: "{{.OUTPUT_DIR}}"
-          OUTPUT_FILE: "{{.CHECKSUM_FILE}}"
-
   meteor:
     vars:
       CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5"

components/core/tools/docker-images/clp-env-base-centos7.4/Dockerfile

@@ -18,6 +18,12 @@ RUN ln -s /opt/rh/devtoolset-10/enable /etc/profile.d/devtoolset.sh
 #       cannot be forced to use a bash shell that loads .bashrc
 RUN cp ./tools/docker-images/clp-env-base-centos7.4/setup-scripts/git /usr/bin/git
 
+# NOTE:
+# 1. `task` doesn't have an apt/yum package so we use its install script.
+# 2. We don't want to install it using `install-prebuilt-packages.sh` since users may use that on
+#    their own machines and it would change their environment in a way that can't easily be undone.
+RUN sh -c "$(curl --location https://taskfile.dev/install.sh)" -- -d -b /usr/local/bin
+
 # Remove cached files
 RUN yum clean all \
     && rm -rf /tmp/* /var/tmp/*

components/core/tools/docker-images/clp-env-base-ubuntu-focal/Dockerfile

@@ -16,6 +16,12 @@ RUN update-alternatives --set cpp /usr/bin/cpp-10
 
 RUN ./tools/scripts/lib_install/ubuntu-focal/install-packages-from-source.sh
 
+# NOTE:
+# 1. `task` doesn't have an apt/yum package so we use its install script.
+# 2. We don't want to install it using `install-prebuilt-packages.sh` since users may use that on
+#    their own machines and it would change their environment in a way that can't easily be undone.
+RUN sh -c "$(curl --location https://taskfile.dev/install.sh)" -- -d -b /usr/local/bin
+
 # Remove cached files
 RUN apt-get clean \
     && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*

components/core/tools/docker-images/clp-env-base-ubuntu-jammy/Dockerfile

@@ -7,6 +7,12 @@ ADD ./tools/scripts/lib_install ./tools/scripts/lib_install
 
 RUN ./tools/scripts/lib_install/ubuntu-jammy/install-all.sh
 
+# NOTE:
+# 1. `task` doesn't have an apt/yum package so we use its install script.
+# 2. We don't want to install it using `install-prebuilt-packages.sh` since users may use that on
+#    their own machines and it would change their environment in a way that can't easily be undone.
+RUN sh -c "$(curl --location https://taskfile.dev/install.sh)" -- -d -b /usr/local/bin
+
 # Remove cached files
 RUN apt-get clean \
     && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*