Merge pull request #158 from HerodotusDev/feat/local-path-request

resolving #157 

1. accept request format like https://github.com/HerodotusDev/hdp-test/blob/feat/local-path-request/requests/9.json
2. with given request from local path, can run via docker image that binded with latest hdp-cairo image

.github/workflows/docker_build_deploy.yml

@@ -0,0 +1,87 @@
+name: Build and Push Docker Images
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - dev
+
+jobs:
+  build-and-push-images:
+    # Add this condition to check for the label
+    runs-on: [self-hosted]
+    permissions:
+      contents: write
+      id-token: write
+    # Add this condition to check for the skip label
+    if: "!contains(github.event.head_commit.message, '[skip build]')"
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      # - name: Log in to Docker Hub
+      #   uses: docker/login-action@v3
+      #   with:
+      #     username: ${{ secrets.DOCKERHUB_USERNAME }}
+      #     password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      # - name: Create remote builder certificates
+      #   run: |
+      #     echo "${{ secrets.BUILDKIT_CA_PEM }}" > ${{ github.workspace }}/client-ca.pem
+      #     echo "${{ secrets.BUILDKIT_CERT_PEM }}" > ${{ github.workspace }}/client-cert.pem
+      #     echo "${{ secrets.BUILDKIT_KEY_PEM }}" > ${{ github.workspace }}/client-key.pem
+
+      # - name: Set up Docker Buildx
+      #   uses: docker/setup-buildx-action@v3
+      #   with:
+      #     driver: remote
+      #     endpoint: "tcp://buildkit.herodotus.dev:5000"
+      #     driver-opts: |
+      #       cacert=${{ github.workspace }}/client-ca.pem
+      #       cert=${{ github.workspace }}/client-cert.pem
+      #       key=${{ github.workspace }}/client-key.pem
+      #     cleanup: true
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v3.1.0
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          version: "lab:latest"
+          driver: cloud
+          endpoint: "dataprocessor/github"
+          cleanup: true
+
+      - name: Fetch HDP & Dry Run for Build
+        run: |
+          ./script/prepare_image_build.sh
+
+      - name: Bump version and push tag
+        id: bump_version
+        run: |
+          git config --local user.email "action@github.com"
+          git config --local user.name "GitHub Action"
+          NEW_VERSION=$(cargo metadata --no-deps --format-version=1 | jq -r '.packages[0].version')
+          echo "Project version is $NEW_VERSION"
+          echo "NEW_VERSION=${NEW_VERSION}" >> $GITHUB_ENV
+          git push --follow-tags
+
+      - name: Build and push Runner Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: runner.dockerfile
+          push: true
+          platforms: linux/amd64,linux/arm64
+          tags: |
+            dataprocessor/hdp-runner:latest
+            dataprocessor/hdp-runner:v${{ env.NEW_VERSION }}
+          cache-from: |
+            type=registry,ref=dataprocessor/hdp-runner:buildcache-amd64,platform=linux/amd64
+            type=registry,ref=dataprocessor/hdp-runner:buildcache-arm64,platform=linux/arm64
+          cache-to: |
+            type=registry,ref=dataprocessor/hdp-runner:buildcache-amd64,mode=max,platform=linux/amd64
+            type=registry,ref=dataprocessor/hdp-runner:buildcache-arm64,mode=max,platform=linux/arm64

.github/workflows/fixtures.yml

@@ -63,24 +63,3 @@ jobs:
           git add .
           git commit -m "Update fixtures"
           git push origin ${{ github.ref_name }}
-
-  merge_to_main:
-    needs: generate_fixtures
-    if: github.ref == 'refs/heads/dev'
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout hdp-test repository
-        uses: actions/checkout@v4
-        with:
-          repository: HerodotusDev/hdp-test
-          token: ${{ secrets.REPO_ACCESS_TOKEN }}
-          fetch-depth: 0
-
-      - name: Merge fixtures to main
-        run: |
-          git config user.name github-actions
-          git config user.email github-actions@github.com
-          git fetch origin
-          git checkout main
-          git merge --no-ff origin/${{ github.ref_name }} -m "Merge ${{ github.ref_name }} into main"
-          git push origin main

cli/src/cli.rs

@@ -11,6 +11,7 @@ use anyhow::Result;
 use clap::Parser;
 use hdp::primitives::processed_types::cairo_format::query::ProcessorInput;
 use hdp::primitives::request::{SubmitBatchQuery, Task};
+use hdp::primitives::task::module::Module;
 use hdp::processor::{self, Processor};
 use hdp::{
     hdp_run,
@@ -101,14 +102,13 @@ pub async fn module_entry_run(args: RunModuleArgs) -> Result<()> {
         args.batch_proof_file,
         args.cairo_pie_file,
     );
+    let module = Module::new_from_str(
+        args.program_hash,
+        args.local_class_path,
+        args.module_inputs.unwrap_or_default(),
+    )?;
     let module_registry = ModuleRegistry::new();
-    let module = module_registry
-        .get_extended_module_from_class_source_string(
-            args.program_hash,
-            args.local_class_path,
-            args.module_inputs.unwrap_or_default(),
-        )
-        .await?;
+    let module = module_registry.get_extended_module(module).await?;
     // TODO: for now, we only support one task if its a module
     let tasks = vec![TaskEnvelope::Module(module)];
 
@@ -189,13 +189,7 @@ pub async fn entry_run(args: RunArgs) -> Result<()> {
                 task_envelopes.push(TaskEnvelope::DatalakeCompute(task));
             }
             Task::Module(task) => {
-                let module = module_registry
-                    .get_extended_module_from_class_source(
-                        Some(task.program_hash),
-                        None,
-                        task.inputs,
-                    )
-                    .await?;
+                let module = module_registry.get_extended_module(task).await?;
                 task_envelopes.push(TaskEnvelope::Module(module));
             }
         }

cli/src/interactive.rs

@@ -2,6 +2,7 @@ use alloy::primitives::U256;
 use anyhow::bail;
 use hdp::hdp_run;
 use hdp::preprocessor::module_registry::ModuleRegistry;
+use hdp::primitives::task::module::Module;
 use hdp::primitives::ChainId;
 use hdp::primitives::{
     aggregate_fn::{integer::Operator, FunctionContext},
@@ -296,14 +297,9 @@ pub async fn run_interactive() -> anyhow::Result<()> {
             .map(|s| s.parse().unwrap())
             .collect();
 
+            let module = Module::new_from_str(Some(module_program_hash), None, module_inputs)?;
             let module_registry = ModuleRegistry::new();
-            let module = module_registry
-                .get_extended_module_from_class_source_string(
-                    Some(module_program_hash),
-                    None,
-                    module_inputs,
-                )
-                .await?;
+            let module = module_registry.get_extended_module(module).await?;
 
             vec![TaskEnvelope::Module(module)]
         }

config/config.json

@@ -0,0 +1,4 @@
+{
+  "HDP_PROGRAM_HASH": "0x01f77febac1a08346546520d148ca8beb99cbb9b26f2f3030d14934003f1f88b",
+  "DRY_RUN_PROGRAM_HASH": "0x048ac124e876e38ec61c5cd1543930e8211d17be84fd37e6c65da472f6801529"
+}

docker-compose.yml

@@ -3,10 +3,13 @@ services:
     build:
       context: .
       dockerfile: ./runner.dockerfile
+      args:
+        HDP_PROGRAM_HASH: 0x0581f639627ec3af482532a1847f59b6521418b59cbe5e2a5657196e3ed886ea
+        DRY_RUN_PROGRAM_HASH: 0x062a6397fa977a8e25887000cef90053ca599e3615830dfeb4cc18f79e534afc
     env_file:
       - .env
     volumes:
       - ./input.json:/hdp-runner/input.json
       - ./output.json:/hdp-runner/output.json
       - ./cairo.pie:/hdp-runner/cairo.pie
-      - ./request.json:/hdp-runner/request.json
+      - ./request.json:/hdp-runner/request.json

examples/private-input-module/src/main.rs

@@ -1,10 +1,7 @@
 use hdp::{
     hdp_run::HdpRunConfig,
     preprocessor::module_registry::ModuleRegistry,
-    primitives::task::{
-        module::{ModuleInput, Visibility},
-        TaskEnvelope,
-    },
+    primitives::task::{module::Module, TaskEnvelope},
 };
 
 #[tokio::main]
@@ -13,17 +10,19 @@ async fn main() {
     std::env::set_var("RUST_LOG", "debug");
 
     let module_regisry = ModuleRegistry::new();
-    let module = module_regisry
-        .get_extended_module_from_class_source(
-            None,
-            Some("./private_module/target/dev/private_module_get_balance.compiled_contract_class.json".into()),
-            vec![
-                ModuleInput::new(Visibility::Private, "0x5222a4"),
-                ModuleInput::new(Visibility::Public, "0x00000000000000000000000013cb6ae34a13a0977f4d7101ebc24b87bb23f0d5" )
-            ],
-        )
-        .await
-        .unwrap();
+    let module = Module::new_from_str(
+        None,
+        Some(
+            "./private_module/target/dev/private_module_get_balance.compiled_contract_class.json"
+                .into(),
+        ),
+        vec![
+            "private.0x5222a4".to_string(),
+            "public.0x00000000000000000000000013cb6ae34a13a0977f4d7101ebc24b87bb23f0d5".to_string(),
+        ],
+    )
+    .unwrap();
+    let module = module_regisry.get_extended_module(module).await.unwrap();
     let tasks = vec![TaskEnvelope::Module(module)];
     let pre_processor_output_file = "input.json";
     let output_file = "output.json";