Merge pull request #15 from openearthplatforminitiative/fix/dask-farg…

…ate-use-private-ip

Fix/dask fargate use private ip

.github/workflows/continous-integration.yaml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
@@ -20,13 +20,41 @@ jobs:
         with:
           context: .
           push: false
+
   lint:
     runs-on: ubuntu-latest
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Check linting
         uses: rickstaa/action-black@v1
         with:
           black_args: ". --check"
+
+  test:
+    runs-on: ubuntu-latest
+    env:
+      # The following environment variables are set to avoid pydantic raising a validation error when running tests.
+      # They are not actually used for running the tests.
+      BASE_DATA_PATH: "test"
+      AWS_REGION: "test"
+      AWS_ACCESS_KEY_ID: "test"
+      AWS_SECRET_ACCESS_KEY: "test"
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install Poetry
+        run: curl -sSL https://install.python-poetry.org | python3 -
+
+      - name: Install dependencies
+        run: poetry install --no-root
+
+      - name: Run pytest
+        run: poetry run pytest

.github/workflows/main-release.yml

@@ -10,20 +10,20 @@ env:
   IMAGE_NAME: ${{ github.repository }}
   # The following environment variables are set to avoid pydantic raising a validation error when running tests.
   # They are not actually used for running the tests.
-  base_data_path:
-  aws_region:
-  aws_access_key_id:
-  aws_secret_access_key:
+  BASE_DATA_PATH: "test"
+  AWS_REGION: "test"
+  AWS_ACCESS_KEY_ID: "test"
+  AWS_SECRET_ACCESS_KEY: "test"
 
 jobs:
   test:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.11'
 

data_pipelines/resources/dask_resource.py

@@ -75,6 +75,7 @@ def _provision_cluster(self, context: InitResourceContext):
                 task_role_arn=self.task_role_arn,
                 security_groups=self.security_groups,
                 skip_cleanup=True,
+                fargate_use_private_ip=True,
             ) as cluster:
                 yield cluster
         else:
@@ -86,5 +87,6 @@ def _provision_cluster(self, context: InitResourceContext):
                 region_name=self.region_name,
                 image=self.image,
                 task_role_policies=self.task_role_policies,
+                fargate_use_private_ip=True,
             ) as cluster:
                 yield cluster