Merge pull request #44 from CodeForAfrica/chore-optimize-image-build

Optimizing Python + Docker deploys using Pants

.github/workflows/twoops-tracker-cd.yml

@@ -4,7 +4,7 @@ on:
     branches:
       - main
     paths:
-      - 'twoops_tracker/py/VERSION'
+      - "twoops_tracker/py/VERSION"
 env:
   DOKKU_REMOTE_BRANCH: "master"
   DOKKU_REMOTE_URL: "ssh://dokku@dokku-1.dev.codeforafrica.org/twoopstracker"
@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.10.7",]
+        python-version: ["3.10"]
     steps:
       - name: Cancel Previous Runs
         uses: styfle/cancel-workflow-action@0.11.0
@@ -32,17 +32,23 @@ jobs:
       - name: Set env
         run: echo "VERSION=$(cat twoops_tracker/py/VERSION)" >> $GITHUB_ENV
 
-      - name: Initialize Pants
-        uses: pantsbuild/actions/init-pants@main
+      - name: Setup Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v1
         with:
-          pants-python-version: ${{ matrix.python-version }}
-          pants-ci-config: "pants.toml"
+          python-version: ${{ matrix.python-version }}
+
+      - name: Bootstrap Pants
+        run: |
+          ./pants --version
+
+      - name: Check BUILD files
+        run: ./pants tailor --check update-build-files --check
 
       - name: Lint
         run: |
-          ./pants lint twoops_tracker/py::
+          ./pants lint --lint-skip-formatters twoops_tracker/::
 
-      - name: Build
+      - name: Build Docker image
         run: |
           VERSION=${{ env.VERSION }} ./pants package twoops_tracker/docker/::
 
@@ -51,7 +57,7 @@ jobs:
         with:
           username: ${{ secrets.DOCKER_HUB_USERNAME }}
           password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
-      
+
       - name: Push to DockerHub
         run: |
           docker push ${{ env.IMAGE_NAME }}:${{ env.VERSION }}

.github/workflows/twoops-tracker-ci.yml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.10.7", ]
+        python-version: ["3.10"]
     steps:
       - name: Cancel Previous Runs
         uses: styfle/cancel-workflow-action@0.11.0
@@ -20,16 +20,22 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Initialize Pants
-        uses: pantsbuild/actions/init-pants@main
+      - name: Setup Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v1
         with:
-          pants-python-version: ${{ matrix.python-version }}
-          pants-ci-config: "pants.toml"
+          python-version: ${{ matrix.python-version }}
+
+      - name: Bootstrap Pants
+        run: |
+          ./pants --version
+
+      - name: Check BUILD files
+        run: ./pants tailor --check update-build-files --check
 
       - name: Lint
         run: |
-          ./pants lint twoops_tracker/py::
+          ./pants lint --lint-skip-formatters twoops_tracker/::
 
       - name: Build
         run: |
-          ./pants package twoops_tracker/py:main
+          ./pants package twoops_tracker/py:twoopstracker-deps twoops_tracker/py:twoopstracker-srcs

README.md

@@ -1 +1,13 @@
-# Commons API
+# COMMONS API
+
+## Applications
+
+| Name                                            | Description                                  |
+| ----------------------------------------------- | -------------------------------------------- |
+| [**TwoopsTracker**](./twoops_tracker/README.md) | A Twitter-based disinformation tracking tool |
+
+## Packages
+
+| Name                               | Description                                                    |
+| ---------------------------------- | -------------------------------------------------------------- |
+| [**`dj-cmd`**](./dj-cmd/README.md) | Command-line utility for running Django when bundled in `pex`. |

contrib/docker-compose/nginx/default.conf

@@ -1,13 +1,13 @@
-upstream twoops_tracker {
-    server twoops_tracker:8000;
+upstream twoopstracker_app {
+    server twoopstracker_app:8000;
 }
 
 server {
 
     listen 80;
 
     location / {
-        proxy_pass http://twoops_tracker;
+        proxy_pass http://twoopstracker_app;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
         proxy_set_header Host $host;
         proxy_redirect off;

dj-cmd/README.md

@@ -0,0 +1,3 @@
+## `dj-cmd`
+
+Command-line utility for running Django & company entry point tasks when bundled in a single `pex` binary.

dj-cmd/py/BUILD

@@ -1,6 +1,6 @@
 resources(
-    name = "resources",
-    sources = [
+    name="resources",
+    sources=[
         "pyproject.toml",
         "setup.cfg",
         "README.md",
@@ -9,13 +9,13 @@ resources(
 )
 
 python_distribution(
-    name = "dj-cmd",
-    dependencies = [
+    name="dj-cmd",
+    dependencies=[
         "dj-cmd/py/dj_cmd:dj_cmd",
         ":resources",
     ],
-    provides = setup_py(
-        name = "dj-cmd",
+    provides=setup_py(
+        name="dj-cmd",
     ),
-    sdist = False,
+    sdist=False,
 )

dj-cmd/py/dj_cmd/BUILD

@@ -1 +1,6 @@
 python_sources()
+
+pex_binary(
+    name="cmd",
+    entry_point="cmd.py",
+)

docker-compose.yml

@@ -24,30 +24,30 @@ services:
       # Allow Redis access from outside the container
       - "63791:6379"
 
-  twoops_tracker:
-    image: codeforafrica/twoops-tracker:latest
+  twoopstracker_app:
+    image: codeforafrica/twoopstracker:latest
     depends_on:
       - db
       - redis
-    environment:
-      - TWOOPSTRACKER_DATABASE_URL=postgres://${PGUSER:-twoops_tracker}:${PGPASSWORD:-twoops_tracker}@db:5432/${PGDATABASE:-twoops_tracker}
-      - TWOOPSTRACKER_CELERY_BROKER_URL=${TWOOPSTRACKER_CELERY_BROKER_URL:-redis://redis:6379/0}
     env_file:
-      - ./twoops_tracker/.env
+      - ./twoops_tracker/.env.template
     volumes:
+      - app-media:/app/media
       - app-staticfiles:/app/staticfiles
-  
-  nginx:
+
+  twoopstracker:
     image: nginx:1.20
     restart: unless-stopped
     volumes:
       - ./contrib/docker-compose/nginx:/etc/nginx/conf.d
+      - app-media:/var/www/media
       - app-staticfiles:/var/www/static
     ports:
       - 8000:80
     depends_on:
-      - twoops_tracker
+      - twoopstracker_app
 
 volumes:
-  db-data:
+  app-media:
   app-staticfiles:
+  db-data:

pants-plugins/BUILD

@@ -1 +1 @@
-pants_requirements(name = "pants")
+pants_requirements(name="pants")

pants-plugins/internal_plugins/versioned_setup/register.py

@@ -49,9 +49,9 @@ async def setup_kwargs_plugin(request: VersionedSetupKwargsRequest) -> SetupKwar
     version_file = kwargs.pop("version_file", None)
     if version and version_file:
         raise ValueError(
-            f"The python_distribution target {request.target.address} has supplied both "
-            "`version` and `version_file` in its setup_py's kwargs. Only one of these "
-            "should be supplied."
+            f"The python_distribution target {request.target.address} has supplied both"
+            " `version` and `version_file` in its setup_py's kwargs. Only one of these"
+            " should be supplied."
         )
     # we default to checking VERSION file if both version and version_file are not set
     if not version:
@@ -62,7 +62,9 @@ async def setup_kwargs_plugin(request: VersionedSetupKwargsRequest) -> SetupKwar
             DigestContents,
             PathGlobs(
                 [version_path],
-                description_of_origin=f"the 'version_file' kwarg in {request.target.address}",
+                description_of_origin=(
+                    f"the 'version_file' kwarg in {request.target.address}"
+                ),
                 glob_match_error_behavior=GlobMatchErrorBehavior.error,
             ),
         )

pants.toml

@@ -1,7 +1,7 @@
 [GLOBAL]
-pants_version = "2.10.0"
-use_deprecated_python_macros = false
-
+pants_version = "2.13.0"
+use_deprecated_directory_cli_args_semantics = false
+use_deprecated_pex_binary_run_semantics = false
 pythonpath = ["%(buildroot)s/pants-plugins"]
 
 backend_packages = [

twoops_tracker/.env.template

@@ -1,27 +1,46 @@
+# This is the exhaustive list of all environment variables used by
+# TwoopsTracker. If a variable is not declared here, docker-compose
+# will not pass it to the container at runtime.
+# 
+# Do not assign values here. This *MUST* remain just a list.
+# Do not even do empty assignments e.g.
+# 
+# TWOOPSTRACKER_DATABASE_URL=
+# 
+# If you do the above, TWOOPSTRACKER_DATABASE_URL will be set to empty
+# string even when a different --env-file is passed to docker-compose
+
 # Required
-TWOOPSTRACKER_DATABASE_URL=
-TWOOPSTRACKER_SECRET_KEY=
-TWOOPSTRACKER_CONSUMER_KEY=
-TWOOPSTRACKER_CONSUMER_SECRET=
-TWOOPSTRACKER_ACCESS_TOKEN=
-TWOOPSTRACKER_ACCESS_TOKEN_SECRET=
-TWOOPTRACKER_STREAM_LISTENER_INTERVAL=
+TWOOPSTRACKER_ACCESS_TOKEN
+TWOOPSTRACKER_ACCESS_TOKEN_SECRET
+TWOOPSTRACKER_CELERY_BROKER_URL
+TWOOPSTRACKER_CONSUMER_KEY
+TWOOPSTRACKER_CONSUMER_SECRET
+TWOOPSTRACKER_DATABASE_URL
+TWOOPSTRACKER_GOOGLE_OAUTH2_CLIENT_ID
+TWOOPSTRACKER_GOOGLE_OAUTH2_CLIENT_SECRET
+TWOOPSTRACKER_SECRET_KEY
+TWOOPSTRACKER_STREAM_LISTENER_INTERVAL
 
 # End of Required
 
 # Optional
-TWOOPSTRACKER_ALLOWED_HOSTS=
-TWOOPSTRACKER_DEBUG=False
-TWOOPSTRACKER_CORS_ALLOWED_ORIGINS=
-TWOOPSTRACKER_SENTRY_DSN=
-TWOOPSTRACKER_EMAIL_CONFIRMATION_URL=
-TWOOPSTRACKER_PASSWORD_RESET_URL=
-ACCOUNT_EMAIL_VERIFICATION=
-SOCIALACCOUNT_EMAIL_VERIFICATION =
-LOGIN_URL=
-
-DEFAULT_FROM_EMAIL=
-EMAIL_USE_TLS=
-EMAIL_PORT=
-EMAIL_HOST_USER=
-EMAIL_HOST_PASSWORD=
+TWOOPSTRACKER_ACCOUNT_EMAIL_VERIFICATION
+TWOOPSTRACKER_ALLOWED_HOSTS
+TWOOPSTRACKER_BASE_DIR
+TWOOPSTRACKER_CORS_ALLOWED_ORIGINS
+TWOOPSTRACKER_DEBUG
+TWOOPSTRACKER_DEFAULT_FROM_EMAIL
+TWOOPSTRACKER_EMAIL_HOST_USER
+TWOOPSTRACKER_EMAIL_HOST_PASSWORD
+TWOOPSTRACKER_EMAIL_PORT
+TWOOPSTRACKER_EMAIL_USE_TLS
+TWOOPSTRACKER_EMAIL_CONFIRMATION_URL
+TWOOPSTRACKER_GUNICORN_EXTRA_CONFIG
+TWOOPSTRACKER_GUNICORN_LOG_LEVEL
+TWOOPSTRACKER_GUNICORN_TIMEOUT
+TWOOPSTRACKER_GUNICORN_WORKERS
+TWOOPSTRACKER_LOGIN_URL
+TWOOPSTRACKER_PASSWORD_RESET_URL
+TWOOPSTRACKER_SENTRY_DSN
+TWOOPSTRACKER_SOCIALACCOUNT_EMAIL_VERIFICATION

twoops_tracker/README.md

@@ -0,0 +1,50 @@
+## TwoopsTracker
+
+A Twitter-based disinformation tracking tool built on a web-based dashboard that collects deleted tweet content from previously identified trolls and disinformation actors. The project seeks to help monitor the social posts of known disinfo actors. The primary tangible output of the project is to expose trolls behind toxic disinformation campaigns who routinely cover their tracks by deleting original inflammatory social media posts that sparked hate speech, disinformation campaigns or conspiracy theories.
+
+## Getting Started
+
+First create `.env` file in the app directory. From project root directory,
+
+```sh
+cp  twoops_tracker/.env.template twoops_tracker/.env
+```
+
+and modify the `.env` file according to your needs.
+
+## Build
+
+To build a pex binary, run:
+
+```sh
+./pants package twoops_tracker/py:twoopstracker
+```
+
+To build the docker image, run:
+
+```sh
+VERSION=$(cat twoops_tracker/py/VERSION) ./pants package twoops_tracker/docker:twoopstracker
+```
+
+## Run
+
+To run pex binary, execute:
+
+```sh
+./pants run twoops_tracker/py:twoopstracker
+```
+
+To run the built docker image, execute:
+
+```sh
+docker-compose --env-file ./twoops_tracker/.env up twoopstracker
+```
+
+**NOTE**: You may need to run `postres` container first to make sure database
+is ready to receive connections _before_ starting the `twoopstracker` app.
+
+To do so, run:
+
+```sh
+docker-compose --env-file ./twoops_tracker/.env up db
+```

twoops_tracker/contrib/dokku/BUILD

@@ -0,0 +1,3 @@
+docker_image(
+    name="docker",
+)

twoops_tracker/contrib/dokku/Dockerfile

@@ -1 +1 @@
-FROM codeforafrica/twoopstracker:2.0.0a5
+FROM codeforafrica/twoopstracker:2.0.0a6