Optimizing Python + Docker deploys using Pants

.github/workflows/twoops-tracker-cd.yml

@@ -40,7 +40,7 @@ jobs:
 
       - name: Lint
         run: |
-          ./pants lint twoops_tracker/py::
+          ./pants lint --lint-skip-formatters twoops_tracker/::
 
       - name: Build
         run: |

.github/workflows/twoops-tracker-ci.yml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.10.7", ]
+        python-version: ["3.10.7"]
     steps:
       - name: Cancel Previous Runs
         uses: styfle/cancel-workflow-action@0.10.1
@@ -28,8 +28,8 @@ jobs:
 
       - name: Lint
         run: |
-          ./pants lint twoops_tracker/py::
+          ./pants lint --lint-skip-formatters twoops_tracker/::
 
       - name: Build
         run: |
-          ./pants package twoops_tracker/py:main
+          ./pants package twoops_tracker/py:twoopstracker-deps twoops_tracker/py:twoopstracker-srcs

README.md

@@ -1 +1,13 @@
-# Commons API
+# COMMONS API
+
+## Applications
+
+| Name                                            | Description                                  |
+| ----------------------------------------------- | -------------------------------------------- |
+| [**TwoopsTracker**](./twoops_tracker/README.md) | A Twitter-based disinformation tracking tool |
+
+## Packages
+
+| Name                               | Description                                                    |
+| ---------------------------------- | -------------------------------------------------------------- |
+| [**`dj-cmd`**](./dj-cmd/README.md) | Command-line utility for running Django when bundled in `pex`. |

contrib/docker-compose/nginx/default.conf

@@ -1,13 +1,13 @@
-upstream twoops_tracker {
-    server twoops_tracker:8000;
+upstream twoopstracker_app {
+    server twoopstracker_app:8000;
 }
 
 server {
 
     listen 80;
 
     location / {
-        proxy_pass http://twoops_tracker;
+        proxy_pass http://twoopstracker_app;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
         proxy_set_header Host $host;
         proxy_redirect off;

dj-cmd/README.md

@@ -0,0 +1,3 @@
+## `dj-cmd`
+
+Command-line utility for running Django & company entry point tasks when bundled in a single `pex` binary.

dj-cmd/py/BUILD

@@ -1,6 +1,6 @@
 resources(
-    name = "resources",
-    sources = [
+    name="resources",
+    sources=[
         "pyproject.toml",
         "setup.cfg",
         "README.md",
@@ -9,13 +9,13 @@ resources(
 )
 
 python_distribution(
-    name = "dj-cmd",
-    dependencies = [
+    name="dj-cmd",
+    dependencies=[
         "dj-cmd/py/dj_cmd:dj_cmd",
         ":resources",
     ],
-    provides = setup_py(
-        name = "dj-cmd",
+    provides=setup_py(
+        name="dj-cmd",
     ),
-    sdist = False,
+    sdist=False,
 )

docker-compose.yml

@@ -24,30 +24,30 @@ services:
       # Allow Redis access from outside the container
       - "63791:6379"
 
-  twoops_tracker:
-    image: codeforafrica/twoops-tracker:latest
+  twoopstracker_app:
+    image: codeforafrica/twoopstracker:latest
     depends_on:
       - db
       - redis
-    environment:
-      - TWOOPSTRACKER_DATABASE_URL=postgres://${PGUSER:-twoops_tracker}:${PGPASSWORD:-twoops_tracker}@db:5432/${PGDATABASE:-twoops_tracker}
-      - TWOOPSTRACKER_CELERY_BROKER_URL=${TWOOPSTRACKER_CELERY_BROKER_URL:-redis://redis:6379/0}
     env_file:
-      - ./twoops_tracker/.env
+      - ./twoops_tracker/.env.template
     volumes:
+      - app-media:/app/media
       - app-staticfiles:/app/staticfiles
-  
-  nginx:
+
+  twoopstracker:
     image: nginx:1.20
     restart: unless-stopped
     volumes:
       - ./contrib/docker-compose/nginx:/etc/nginx/conf.d
+      - app-media:/var/www/media
       - app-staticfiles:/var/www/static
     ports:
       - 8000:80
     depends_on:
-      - twoops_tracker
+      - twoopstracker_app
 
 volumes:
-  db-data:
+  app-media:
   app-staticfiles:
+  db-data:

pants-plugins/BUILD

@@ -1 +1 @@
-pants_requirements(name = "pants")
+pants_requirements(name="pants")

pants-plugins/internal_plugins/versioned_setup/register.py

@@ -49,9 +49,9 @@ async def setup_kwargs_plugin(request: VersionedSetupKwargsRequest) -> SetupKwar
     version_file = kwargs.pop("version_file", None)
     if version and version_file:
         raise ValueError(
-            f"The python_distribution target {request.target.address} has supplied both "
-            "`version` and `version_file` in its setup_py's kwargs. Only one of these "
-            "should be supplied."
+            f"The python_distribution target {request.target.address} has supplied both"
+            " `version` and `version_file` in its setup_py's kwargs. Only one of these"
+            " should be supplied."
         )
     # we default to checking VERSION file if both version and version_file are not set
     if not version:
@@ -62,7 +62,9 @@ async def setup_kwargs_plugin(request: VersionedSetupKwargsRequest) -> SetupKwar
             DigestContents,
             PathGlobs(
                 [version_path],
-                description_of_origin=f"the 'version_file' kwarg in {request.target.address}",
+                description_of_origin=(
+                    f"the 'version_file' kwarg in {request.target.address}"
+                ),
                 glob_match_error_behavior=GlobMatchErrorBehavior.error,
             ),
         )

pants.toml

@@ -1,7 +1,7 @@
 [GLOBAL]
-pants_version = "2.10.0"
-use_deprecated_python_macros = false
-
+pants_version = "2.13.0"
+use_deprecated_directory_cli_args_semantics = false
+use_deprecated_pex_binary_run_semantics = false
 pythonpath = ["%(buildroot)s/pants-plugins"]
 
 backend_packages = [

twoops_tracker/.env.template

@@ -1,27 +1,45 @@
+# This is the exhaustive list of all environemtn variables used by
+# TwoopsTracker. If a variable is not declared here, docker-compose
+# will not pass it to container at runtime.
+# 
+# Do not assign values here. This *MUST* remain a just a list.
+# Do not do even empty assignment e.g.
+# 
+# TWOOPSTRACKER_DATABASE_URL=
+# 
+# If you do the above, TWOOPSTRACKER_DATABASE_URL will be set to empty
+# string even if a different --env-file is passed to docker-compose
+
 # Required
-TWOOPSTRACKER_DATABASE_URL=
-TWOOPSTRACKER_SECRET_KEY=
-TWOOPSTRACKER_CONSUMER_KEY=
-TWOOPSTRACKER_CONSUMER_SECRET=
-TWOOPSTRACKER_ACCESS_TOKEN=
-TWOOPSTRACKER_ACCESS_TOKEN_SECRET=
-TWOOPTRACKER_STREAM_LISTENER_INTERVAL=
+TWOOPSTRACKER_ACCESS_TOKEN
+TWOOPSTRACKER_ACCESS_TOKEN_SECRET
+TWOOPSTRACKER_CONSUMER_KEY
+TWOOPSTRACKER_CONSUMER_SECRET
+TWOOPSTRACKER_DATABASE_URL
+TWOOPSTRACKER_GOOGLE_OAUTH2_CLIENT_ID
+TWOOPSTRACKER_GOOGLE_OAUTH2_CLIENT_SECRET
+TWOOPSTRACKER_SECRET_KEY
+TWOOPSTRACKER_STREAM_LISTENER_INTERVAL
 
 # End of Required
 
 # Optional
-TWOOPSTRACKER_ALLOWED_HOSTS=
-TWOOPSTRACKER_DEBUG=False
-TWOOPSTRACKER_CORS_ALLOWED_ORIGINS=
-TWOOPSTRACKER_SENTRY_DSN=
-TWOOPSTRACKER_EMAIL_CONFIRMATION_URL=
-TWOOPSTRACKER_PASSWORD_RESET_URL=
-ACCOUNT_EMAIL_VERIFICATION=
-SOCIALACCOUNT_EMAIL_VERIFICATION =
-LOGIN_URL=
-
-DEFAULT_FROM_EMAIL=
-EMAIL_USE_TLS=
-EMAIL_PORT=
-EMAIL_HOST_USER=
-EMAIL_HOST_PASSWORD=
+TWOOPSTRACKER_ACCOUNT_EMAIL_VERIFICATION
+TWOOPSTRACKER_ALLOWED_HOSTS
+TWOOPSTRACKER_BASE_DIR
+TWOOPSTRACKER_CORS_ALLOWED_ORIGINS
+TWOOPSTRACKER_DEBUG
+TWOOPSTRACKER_DEFAULT_FROM_EMAIL
+TWOOPSTRACKER_EMAIL_HOST_USER
+TWOOPSTRACKER_EMAIL_HOST_PASSWORD
+TWOOPSTRACKER_EMAIL_PORT
+TWOOPSTRACKER_EMAIL_USE_TLS
+TWOOPSTRACKER_EMAIL_CONFIRMATION_URL
+TWOOPSTRACKER_GUNICORN_EXTRA_CONFIG
+TWOOPSTRACKER_GUNICORN_LOG_LEVEL
+TWOOPSTRACKER_GUNICORN_TIMEOUT
+TWOOPSTRACKER_GUNICORN_WORKERS
+TWOOPSTRACKER_LOGIN_URL
+TWOOPSTRACKER_PASSWORD_RESET_URL
+TWOOPSTRACKER_SENTRY_DSN
+TWOOPSTRACKER_SOCIALACCOUNT_EMAIL_VERIFICATION

twoops_tracker/README.md

@@ -0,0 +1,50 @@
+## TwoopsTracker
+
+A Twitter-based disinformation tracking tool built on a web-based dashboard that collects deleted tweet content from previously identified trolls and disinformation actors. The project seeks to help monitor the social posts of known disinfo actors. The primary tangible output of the project is to expose trolls behind toxic disinformation campaigns who routinely cover their tracks by deleting original inflammatory social media posts that sparked hate speech, disinformation campaigns or conspiracy theories.
+
+## Getting Started
+
+First create `.env` file in the app directory. From project root directory,
+
+```sh
+cp  twoops_tracker/.env.template twoops_tracker/.env
+```
+
+and modify the `.env` file according to your needs.
+
+## Build
+
+To build a pex binary, run:
+
+```sh
+./pants package twoops_tracker/py:twoopstracker
+```
+
+To build the docker image, run:
+
+```sh
+VERSION=$(cat twoops_tracker/py/VERSION) ./pants package twoops_tracker/docker:twoopstracker
+```
+
+## Run
+
+To run pex binary, execute:
+
+```sh
+./pants run twoops_tracker/py:twoopstracker
+```
+
+To run the built docker image, execute:
+
+```sh
+docker-compose --env-file ./twoops_tracker/.env up twoopstracker
+```
+
+**NOTE**: You may need to run `postres` container first to make sure database
+is ready to receive connections _before_ starting the `twoopstracker` app.
+
+To do so, run:
+
+```sh
+docker-compose --env-file ./twoops_tracker/.env up db
+```

twoops_tracker/docker/BUILD

@@ -1,8 +1,29 @@
+docker_image(
+    name = "twoopstracker-deps",
+    image_tags = ["deps"],
+    # We just need to hardcode something in the registries so that we can
+    # reference it the final image's COPY instructions. Can be anything
+    registries = ["twoopstracker"],
+    repository = "app",
+    skip_push = True,
+    source = "Dockerfile.deps",
+)
+
+docker_image(
+    name = "twoopstracker-srcs",
+    image_tags = ["srcs"],
+    registries = ["twoopstracker"],
+    repository = "app",
+    skip_push = True,
+    source = "Dockerfile.srcs",
+)
+
 docker_image(
     name = "twoopstracker",
     dependencies = [
-        "twoops_tracker/py:main",
-        "twoops_tracker/sh:sh",
+        ":twoopstracker-deps",
+        ":twoopstracker-srcs",
+        "twoops_tracker/sh/cmd.sh",
     ],
     image_tags = [
         "{build_args.VERSION}",

twoops_tracker/docker/Dockerfile

@@ -1,26 +1,26 @@
-FROM python:3.10-slim
+FROM python:3.10-slim-bullseye AS python-base
 
-### Arg
-ARG DEBIAN_FRONTEND=noninteractive
+RUN apt-get update \
+    && apt-get -y upgrade \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
-### Env
-ENV APP_HOST=.
-ENV APP_DOCKER=/app
-ENV PYTHONDONTWRITEBYTECODE 1
-ENV PYTHONUNBUFFERED 1
-ENV STATIC_ROOT=/app/staticfiles
+FROM twoopstracker/app:deps AS app-deps
 
-COPY twoops_tracker.py/main.pex /bin/dj
+FROM twoopstracker/app:srcs AS app-srcs
 
-# Expose server port
-EXPOSE 8000
+FROM python-base AS python-app
+
+WORKDIR /app
 
-### Volumes
-WORKDIR ${APP_DOCKER}
 RUN mkdir -p media staticfiles logs
 
-COPY twoops_tracker/sh/*.sh /
-RUN chmod +x /cmd.sh
+COPY --from=app-deps /app ./
+COPY --from=app-srcs /app ./
+
+COPY twoops_tracker/sh/cmd.sh ./
+RUN chmod +x ./cmd.sh
+
+EXPOSE 8000
 
-### Run app
-CMD ["/cmd.sh"]
+ENTRYPOINT ["./cmd.sh"]

twoops_tracker/docker/Dockerfile.deps

@@ -0,0 +1,4 @@
+FROM python:3.10-slim-bullseye
+
+COPY twoops_tracker.py/twoopstracker-deps.pex /twoopstracker-deps.pex
+RUN PEX_TOOLS=1 /usr/local/bin/python3.10 /twoopstracker-deps.pex venv --scope=deps --compile /app

twoops_tracker/docker/Dockerfile.srcs

@@ -0,0 +1,4 @@
+FROM python:3.10-slim-bullseye
+
+COPY twoops_tracker.py/twoopstracker-srcs.pex /twoopstracker-srcs.pex
+RUN PEX_TOOLS=1 /usr/local/bin/python3.10 /twoopstracker-srcs.pex venv --scope=srcs --compile /app