From aaea0cbd6af605d7a557c7d9a5209da3eb250641 Mon Sep 17 00:00:00 2001 From: esir Date: Wed, 5 Oct 2022 17:03:39 +0300 Subject: [PATCH 01/22] include_tools while building main pex binary --- twoops_tracker/py/BUILD | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/twoops_tracker/py/BUILD b/twoops_tracker/py/BUILD index b90d2756..1acadbe0 100644 --- a/twoops_tracker/py/BUILD +++ b/twoops_tracker/py/BUILD @@ -22,8 +22,11 @@ python_sources( pex_binary( name = "main", + entry_point = "main.py", dependencies = [ ":lib", ], - entry_point = "main.py", + layout = "packed", + execution_mode = "venv", + include_tools = True, ) From ea7f26c2a13c673bf58f35c08d4e11538cd36810 Mon Sep 17 00:00:00 2001 From: esir Date: Wed, 5 Oct 2022 17:04:22 +0300 Subject: [PATCH 02/22] Use multi stage build to extract pex to run commands --- twoops_tracker/docker/Dockerfile | 28 +++++++++++++++++----------- twoops_tracker/sh/cmd.sh | 10 +++++----- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/twoops_tracker/docker/Dockerfile b/twoops_tracker/docker/Dockerfile index fd9cc32c..e7e186dc 100644 --- a/twoops_tracker/docker/Dockerfile +++ b/twoops_tracker/docker/Dockerfile @@ -1,26 +1,32 @@ -FROM python:3.10-slim +FROM python:3.10-slim as deps -### Arg -ARG DEBIAN_FRONTEND=noninteractive +COPY twoops_tracker.py/main.pex /main.pex +RUN PEX_TOOLS=1 /usr/local/bin/python3.10 /main.pex venv --scope=deps --compile /bin/dj + + +FROM python:3.10-slim as srcs + +COPY twoops_tracker.py/main.pex /main.pex +RUN PEX_TOOLS=1 /usr/local/bin/python3.10 /main.pex venv --scope=srcs --compile /bin/dj -### Env -ENV APP_HOST=. -ENV APP_DOCKER=/app -ENV PYTHONDONTWRITEBYTECODE 1 -ENV PYTHONUNBUFFERED 1 -ENV STATIC_ROOT=/app/staticfiles -COPY twoops_tracker.py/main.pex /bin/dj +FROM python:3.10-slim + +ENV APP_DOCKER=/app # Expose server port EXPOSE 8000 ### Volumes WORKDIR ${APP_DOCKER} + RUN mkdir -p media staticfiles logs COPY twoops_tracker/sh/*.sh / RUN chmod +x /cmd.sh +COPY --from=deps /bin/dj /bin/dj +COPY --from=srcs /bin/dj /bin/dj + ### Run app -CMD ["/cmd.sh"] +ENTRYPOINT ["/cmd.sh"] diff --git a/twoops_tracker/sh/cmd.sh b/twoops_tracker/sh/cmd.sh index 167055e6..73b76af0 100644 --- a/twoops_tracker/sh/cmd.sh +++ b/twoops_tracker/sh/cmd.sh @@ -1,6 +1,6 @@ #!/bin/sh -dj manage migrate --noinput # Apply database migrations -dj manage collectstatic --noinput # Collect static files +/bin/dj/pex manage migrate --noinput # Apply database migrations +/bin/dj/pex manage collectstatic --noinput # Collect static files # Prepare log files and start outputting logs to stdout touch /app/logs/gunicorn.log @@ -10,16 +10,16 @@ touch /app/logs/celery.log tail -n 0 -f /app/logs/*.log & # Start celery worker -dj celery -A twoopstracker worker -l INFO >/dev/null 2>/app/logs/celery.log & +/bin/dj/pex celery -A twoopstracker worker -l INFO >/dev/null 2>/app/logs/celery.log & # everytime the container is restarted, the scheduler will reset rm -rf celerybeat-schedule # Start celery beat service -dj celery -A twoopstracker beat -l INFO >/dev/null 2>/app/logs/celery.log & +/bin/dj/pex celery -A twoopstracker beat -l INFO >/dev/null 2>/app/logs/celery.log & # Start Gunicorn processes echo Starting Gunicorn. -exec dj gunicorn \ +exec /bin/dj/pex gunicorn \ --bind 0.0.0.0:8000 \ --workers="${TWOOPS_TRACKER_GUNICORN_WORKERS:-3}" \ --worker-class gevent \ From ec5df2a2ec06c65d7fd89095fb2c829093c93aee Mon Sep 17 00:00:00 2001 From: esir Date: Wed, 5 Oct 2022 17:05:54 +0300 Subject: [PATCH 03/22] Remove STATIC_ROOT as an environment variable --- twoops_tracker/py/twoopstracker/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/twoops_tracker/py/twoopstracker/settings.py b/twoops_tracker/py/twoopstracker/settings.py index d4396156..28315e1c 100644 --- a/twoops_tracker/py/twoopstracker/settings.py +++ b/twoops_tracker/py/twoopstracker/settings.py @@ -159,7 +159,7 @@ # https://docs.djangoproject.com/en/3.2/howto/static-files/ STATIC_URL = "/static/" -STATIC_ROOT = env.str("STATIC_ROOT", BASE_DIR / "staticfiles") +STATIC_ROOT = BASE_DIR / "staticfiles" # Default primary key field type # https://docs.djangoproject.com/en/3.2/ref/settings/#default-auto-field From 2f28a47909345e86c143b4c32f31dd2ee7f17b88 Mon Sep 17 00:00:00 2001 From: _ Kilemensi Date: Mon, 10 Oct 2022 16:31:59 +0300 Subject: [PATCH 04/22] Upgrade pants --- pants.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pants.toml b/pants.toml index eba82d9c..0909a69f 100644 --- a/pants.toml +++ b/pants.toml @@ -1,7 +1,7 @@ [GLOBAL] -pants_version = "2.10.0" -use_deprecated_python_macros = false - +pants_version = "2.13.0" +use_deprecated_directory_cli_args_semantics = false +use_deprecated_pex_binary_run_semantics = false pythonpath = ["%(buildroot)s/pants-plugins"] backend_packages = [ From f5e80039a63c7f6ea9f05608a65c0dbb21cf714e Mon Sep 17 00:00:00 2001 From: _ Kilemensi Date: Mon, 10 Oct 2022 16:32:44 +0300 Subject: [PATCH 05/22] Use multip images and tags in building twoopstracker --- twoops_tracker/docker/BUILD | 25 ++++++++++++++++-- twoops_tracker/docker/Dockerfile | 38 +++++++++++---------------- twoops_tracker/docker/Dockerfile.deps | 4 +++ twoops_tracker/docker/Dockerfile.srcs | 4 +++ 4 files changed, 47 insertions(+), 24 deletions(-) create mode 100644 twoops_tracker/docker/Dockerfile.deps create mode 100644 twoops_tracker/docker/Dockerfile.srcs diff --git a/twoops_tracker/docker/BUILD b/twoops_tracker/docker/BUILD index 40e7ee71..c20984fc 100644 --- a/twoops_tracker/docker/BUILD +++ b/twoops_tracker/docker/BUILD @@ -1,8 +1,29 @@ +docker_image( + name = "twoopstracker-deps", + image_tags = ["deps"], + # We just need to hardcode something in the registries so that we can + # reference it the final image's COPY instructions. Can be anything + registries = ["twoopstracker"], + repository = "app", + skip_push = True, + source = "Dockerfile.deps", +) + +docker_image( + name = "twoopstracker-srcs", + image_tags = ["srcs"], + registries = ["twoopstracker"], + repository = "app", + skip_push = True, + source = "Dockerfile.srcs", +) + docker_image( name = "twoopstracker", dependencies = [ - "twoops_tracker/py:main", - "twoops_tracker/sh:sh", + ":twoopstracker-deps", + ":twoopstracker-srcs", + "twoops_tracker/sh/cmd.sh", ], image_tags = [ "{build_args.VERSION}", diff --git a/twoops_tracker/docker/Dockerfile b/twoops_tracker/docker/Dockerfile index e7e186dc..ae3dddc4 100644 --- a/twoops_tracker/docker/Dockerfile +++ b/twoops_tracker/docker/Dockerfile @@ -1,32 +1,26 @@ -FROM python:3.10-slim as deps +FROM python:3.10-slim-bullseye AS python-base -COPY twoops_tracker.py/main.pex /main.pex -RUN PEX_TOOLS=1 /usr/local/bin/python3.10 /main.pex venv --scope=deps --compile /bin/dj +RUN apt-get update \ + && apt-get -y upgrade \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* +FROM twoopstracker/app:deps AS app-deps -FROM python:3.10-slim as srcs +FROM twoopstracker/app:srcs AS app-srcs -COPY twoops_tracker.py/main.pex /main.pex -RUN PEX_TOOLS=1 /usr/local/bin/python3.10 /main.pex venv --scope=srcs --compile /bin/dj +FROM python-base AS python-app - -FROM python:3.10-slim - -ENV APP_DOCKER=/app - -# Expose server port -EXPOSE 8000 - -### Volumes -WORKDIR ${APP_DOCKER} +WORKDIR /app RUN mkdir -p media staticfiles logs -COPY twoops_tracker/sh/*.sh / -RUN chmod +x /cmd.sh +COPY --from=app-deps /app ./ +COPY --from=app-srcs /app ./ + +COPY twoops_tracker/sh/cmd.sh ./ +RUN chmod +x ./cmd.sh -COPY --from=deps /bin/dj /bin/dj -COPY --from=srcs /bin/dj /bin/dj +EXPOSE 8000 -### Run app -ENTRYPOINT ["/cmd.sh"] +ENTRYPOINT ["./cmd.sh"] \ No newline at end of file diff --git a/twoops_tracker/docker/Dockerfile.deps b/twoops_tracker/docker/Dockerfile.deps new file mode 100644 index 00000000..0f0efa8d --- /dev/null +++ b/twoops_tracker/docker/Dockerfile.deps @@ -0,0 +1,4 @@ +FROM python:3.10-slim-bullseye + +COPY twoops_tracker.py/twoopstracker-deps.pex /twoopstracker-deps.pex +RUN PEX_TOOLS=1 /usr/local/bin/python3.10 /twoopstracker-deps.pex venv --scope=deps --compile /app diff --git a/twoops_tracker/docker/Dockerfile.srcs b/twoops_tracker/docker/Dockerfile.srcs new file mode 100644 index 00000000..5785fb75 --- /dev/null +++ b/twoops_tracker/docker/Dockerfile.srcs @@ -0,0 +1,4 @@ +FROM python:3.10-slim-bullseye + +COPY twoops_tracker.py/twoopstracker-srcs.pex /twoopstracker-srcs.pex +RUN PEX_TOOLS=1 /usr/local/bin/python3.10 /twoopstracker-srcs.pex venv --scope=srcs --compile /app From fb891815cfbd0931183e3c28e666f2fd4e8e9f33 Mon Sep 17 00:00:00 2001 From: _ Kilemensi Date: Mon, 10 Oct 2022 16:34:01 +0300 Subject: [PATCH 06/22] Run script from current directory only --- twoops_tracker/sh/cmd.sh | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/twoops_tracker/sh/cmd.sh b/twoops_tracker/sh/cmd.sh index 73b76af0..d4c33f4f 100644 --- a/twoops_tracker/sh/cmd.sh +++ b/twoops_tracker/sh/cmd.sh @@ -1,32 +1,34 @@ #!/bin/sh -/bin/dj/pex manage migrate --noinput # Apply database migrations -/bin/dj/pex manage collectstatic --noinput # Collect static files +./pex manage migrate --noinput # Apply database migrations +./pex manage collectstatic --noinput # Collect static files # Prepare log files and start outputting logs to stdout -touch /app/logs/gunicorn.log -touch /app/logs/access.log -touch /app/logs/celery.log +touch ./logs/gunicorn.log +touch ./logs/access.log +touch ./logs/celery.log -tail -n 0 -f /app/logs/*.log & +tail -n 0 -f ./logs/*.log & # Start celery worker -/bin/dj/pex celery -A twoopstracker worker -l INFO >/dev/null 2>/app/logs/celery.log & +echo Starting celery worker +./pex celery -A twoopstracker worker -l INFO >/dev/null 2>./logs/celery.log & # everytime the container is restarted, the scheduler will reset rm -rf celerybeat-schedule # Start celery beat service -/bin/dj/pex celery -A twoopstracker beat -l INFO >/dev/null 2>/app/logs/celery.log & +echo Starting celery beat +./pex celery -A twoopstracker beat -l INFO >/dev/null 2>./logs/celery.log & # Start Gunicorn processes -echo Starting Gunicorn. -exec /bin/dj/pex gunicorn \ +echo Starting gunicorn +exec ./pex gunicorn \ --bind 0.0.0.0:8000 \ - --workers="${TWOOPS_TRACKER_GUNICORN_WORKERS:-3}" \ + --workers="${TWOOPSTRACKER_GUNICORN_WORKERS:-3}" \ --worker-class gevent \ - --log-level="${TWOOPS_GUNICORN_LOG_LEVEL:-warning}" \ - --timeout="${TWOOPS_GUNICORN_TIMEOUT:-60}" \ - --log-file=/app/logs/gunicorn.log \ - --access-logfile=/app/logs/access.log \ - --name twoopsTracker \ - "${TWOOPSTRACKER_GUNICORN_EXTRA_CONFIG:-}" \ + --log-level="${TWOOPSTRACKER_GUNICORN_LOG_LEVEL:-warning}" \ + --timeout="${TWOOPSTRACKER_GUNICORN_TIMEOUT:-60}" \ + --log-file=./logs/gunicorn.log \ + --access-logfile=./logs/access.log \ + --name twoopstracker \ + "${TWOOPSTRACKER_GUNICORN_EXTRA_CONFIG:---reload}" \ twoopstracker.wsgi:application From c2885dfc8f11562f1c6d4f3f425879c0f3583c58 Mon Sep 17 00:00:00 2001 From: _ Kilemensi Date: Mon, 10 Oct 2022 16:34:37 +0300 Subject: [PATCH 07/22] Build separate pex for sources and dependencies --- twoops_tracker/py/BUILD | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/twoops_tracker/py/BUILD b/twoops_tracker/py/BUILD index 1acadbe0..49148463 100644 --- a/twoops_tracker/py/BUILD +++ b/twoops_tracker/py/BUILD @@ -21,12 +21,23 @@ python_sources( ) pex_binary( - name = "main", - entry_point = "main.py", + name = "twoopstracker-deps", dependencies = [ ":lib", ], + entry_point = "main.py", + include_sources = False, + include_tools = True, layout = "packed", - execution_mode = "venv", +) + +pex_binary( + name = "twoopstracker-srcs", + dependencies = [ + ":lib", + ], + entry_point = "main.py", + include_requirements = False, include_tools = True, + layout = "packed", ) From 633364779022122ccfee5a45f1e50a7ed509e50a Mon Sep 17 00:00:00 2001 From: _ Kilemensi Date: Mon, 10 Oct 2022 16:37:06 +0300 Subject: [PATCH 08/22] Allow BASE_DIR to be set to where pex binary is This should allow staticfiles, media, logs, etc., to be in Docker volumes. Also standardize all env vars to start with TWOOPSTRACKER_. --- twoops_tracker/py/twoopstracker/settings.py | 29 ++++++++++++--------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/twoops_tracker/py/twoopstracker/settings.py b/twoops_tracker/py/twoopstracker/settings.py index 28315e1c..f35e9d67 100644 --- a/twoops_tracker/py/twoopstracker/settings.py +++ b/twoops_tracker/py/twoopstracker/settings.py @@ -23,8 +23,11 @@ # Core Settings # https://docs.djangoproject.com/en/3.2/ref/settings/#core-settings + +BASE_DIR = env.path("TWOOPSTRACKER_BASE_DIR", None) +if not BASE_DIR: + BASE_DIR = Path(__file__).resolve().parent.parent # Build paths inside the project like this: BASE_DIR / 'subdir'. -BASE_DIR = Path(__file__).resolve().parent.parent # Sentry SENTRY_DSN = env.str("TWOOPSTRACKER_SENTRY_DSN", "") @@ -178,8 +181,8 @@ # Celery CELERY_BROKER_URL = env.str("TWOOPSTRACKER_CELERY_BROKER_URL", "") -TWOOPTRACKER_STREAM_LISTENER_INTERVAL = env.int( - "TWOOPTRACKER_STREAM_LISTENER_INTERVAL", 15 +TWOOPSTRACKER_STREAM_LISTENER_INTERVAL = env.int( + "TWOOPSTRACKER_STREAM_LISTENER_INTERVAL", 15 ) # Static Files @@ -223,11 +226,13 @@ # some to env vars. ACCOUNT_AUTHENTICATION_METHOD = "email" ACCOUNT_EMAIL_REQUIRED = True -ACCOUNT_EMAIL_VERIFICATION = env.str("ACCOUNT_EMAIL_VERIFICATION", "none") -SOCIALACCOUNT_EMAIL_VERIFICATION = env.str("SOCIALACCOUNT_EMAIL_VERIFICATION", "none") +ACCOUNT_EMAIL_VERIFICATION = env.str("TWOOPSTRACKER_ACCOUNT_EMAIL_VERIFICATION", "none") +SOCIALACCOUNT_EMAIL_VERIFICATION = env.str( + "TWOOPSTRACKER_SOCIALACCOUNT_EMAIL_VERIFICATION", "none" +) ACCOUNT_USER_MODEL_USERNAME_FIELD = None ACCOUNT_USERNAME_REQUIRED = False -LOGIN_URL = env("LOGIN_URL", "http://localhost:3000/login") +LOGIN_URL = env("TWOOPSTRACKER_LOGIN_URL", "http://localhost:3000/login") TWOOPSTRACKER_EMAIL_CONFIRMATION_URL = env.str( "TWOOPSTRACKER_EMAIL_CONFIRMATION_URL", "http://localhost:3000/api/auth/registration/confirm-email", @@ -254,12 +259,12 @@ # https://django-allauth.readthedocs.io/en/latest/advanced.html#custom-redirects ACCOUNT_ADAPTER = "twoopstracker.authentication.adapter.CustomAccountAdapter" -DEFAULT_FROM_EMAIL = env.str("DEFAULT_FROM_EMAIL", "webmaster@localhost") -EMAIL_HOST = env("EMAIL_HOST", "") -EMAIL_USE_TLS = env.bool("EMAIL_USE_TLS", True) -EMAIL_PORT = env.int("EMAIL_PORT", 587) -EMAIL_HOST_USER = env("EMAIL_HOST_USER", "") -EMAIL_HOST_PASSWORD = env("EMAIL_HOST_PASSWORD", "") +DEFAULT_FROM_EMAIL = env.str("TWOOPSTRACKER_DEFAULT_FROM_EMAIL", "webmaster@localhost") +EMAIL_HOST = env("TWOOPSTRACKER_EMAIL_HOST", "") +EMAIL_USE_TLS = env.bool("TWOOPSTRACKER_EMAIL_USE_TLS", True) +EMAIL_PORT = env.int("TWOOPSTRACKER_EMAIL_PORT", 587) +EMAIL_HOST_USER = env("TWOOPSTRACKER_EMAIL_HOST_USER", "") +EMAIL_HOST_PASSWORD = env("TWOOPSTRACKER_EMAIL_HOST_PASSWORD", "") REST_SESSION_LOGIN = False REST_USE_JWT = True From 1b7e3209d9fc2899c16eb7a621045138d33383a8 Mon Sep 17 00:00:00 2001 From: _ Kilemensi Date: Mon, 10 Oct 2022 16:38:40 +0300 Subject: [PATCH 09/22] Update env template to match current usage --- twoops_tracker/.env.template | 62 +++++++++++++++++++++++------------- 1 file changed, 40 insertions(+), 22 deletions(-) diff --git a/twoops_tracker/.env.template b/twoops_tracker/.env.template index 41a15b52..1a218132 100644 --- a/twoops_tracker/.env.template +++ b/twoops_tracker/.env.template @@ -1,27 +1,45 @@ +# This is the exhaustive list of all environemtn variables used by +# TwoopsTracker. If a variable is not declared here, docker-compose +# will not pass it to container at runtime. +# +# Do not assign values here. This *MUST* remain a just a list. +# Do not do even empty assignment e.g. +# +# TWOOPSTRACKER_DATABASE_URL= +# +# If you do the above, TWOOPSTRACKER_DATABASE_URL will be set to empty +# string even if a different --env-file is passed to docker-compose + # Required -TWOOPSTRACKER_DATABASE_URL= -TWOOPSTRACKER_SECRET_KEY= -TWOOPSTRACKER_CONSUMER_KEY= -TWOOPSTRACKER_CONSUMER_SECRET= -TWOOPSTRACKER_ACCESS_TOKEN= -TWOOPSTRACKER_ACCESS_TOKEN_SECRET= -TWOOPTRACKER_STREAM_LISTENER_INTERVAL= +TWOOPSTRACKER_ACCESS_TOKEN +TWOOPSTRACKER_ACCESS_TOKEN_SECRET +TWOOPSTRACKER_CONSUMER_KEY +TWOOPSTRACKER_CONSUMER_SECRET +TWOOPSTRACKER_DATABASE_URL +TWOOPSTRACKER_GOOGLE_OAUTH2_CLIENT_ID +TWOOPSTRACKER_GOOGLE_OAUTH2_CLIENT_SECRET +TWOOPSTRACKER_SECRET_KEY +TWOOPSTRACKER_STREAM_LISTENER_INTERVAL # End of Required # Optional -TWOOPSTRACKER_ALLOWED_HOSTS= -TWOOPSTRACKER_DEBUG=False -TWOOPSTRACKER_CORS_ALLOWED_ORIGINS= -TWOOPSTRACKER_SENTRY_DSN= -TWOOPSTRACKER_EMAIL_CONFIRMATION_URL= -TWOOPSTRACKER_PASSWORD_RESET_URL= -ACCOUNT_EMAIL_VERIFICATION= -SOCIALACCOUNT_EMAIL_VERIFICATION = -LOGIN_URL= - -DEFAULT_FROM_EMAIL= -EMAIL_USE_TLS= -EMAIL_PORT= -EMAIL_HOST_USER= -EMAIL_HOST_PASSWORD= +TWOOPSTRACKER_ACCOUNT_EMAIL_VERIFICATION +TWOOPSTRACKER_ALLOWED_HOSTS +TWOOPSTRACKER_BASE_DIR +TWOOPSTRACKER_CORS_ALLOWED_ORIGINS +TWOOPSTRACKER_DEBUG +TWOOPSTRACKER_DEFAULT_FROM_EMAIL +TWOOPSTRACKER_EMAIL_HOST_USER +TWOOPSTRACKER_EMAIL_HOST_PASSWORD +TWOOPSTRACKER_EMAIL_PORT +TWOOPSTRACKER_EMAIL_USE_TLS +TWOOPSTRACKER_EMAIL_CONFIRMATION_URL +TWOOPSTRACKER_GUNICORN_EXTRA_CONFIG +TWOOPSTRACKER_GUNICORN_LOG_LEVEL +TWOOPSTRACKER_GUNICORN_TIMEOUT +TWOOPSTRACKER_GUNICORN_WORKERS +TWOOPSTRACKER_LOGIN_URL +TWOOPSTRACKER_PASSWORD_RESET_URL +TWOOPSTRACKER_SENTRY_DSN +TWOOPSTRACKER_SOCIALACCOUNT_EMAIL_VERIFICATION From f16417b5c659ede545c6fd42b342fc466dfc0b69 Mon Sep 17 00:00:00 2001 From: _ Kilemensi Date: Mon, 10 Oct 2022 16:39:14 +0300 Subject: [PATCH 10/22] Restructure services to match current usage --- docker-compose.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 2d194d98..43ce7add 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -24,30 +24,30 @@ services: # Allow Redis access from outside the container - "63791:6379" - twoops_tracker: - image: codeforafrica/twoops-tracker:latest + twoopstracker_app: + image: codeforafrica/twoopstracker:latest depends_on: - db - redis - environment: - - TWOOPSTRACKER_DATABASE_URL=postgres://${PGUSER:-twoops_tracker}:${PGPASSWORD:-twoops_tracker}@db:5432/${PGDATABASE:-twoops_tracker} - - TWOOPSTRACKER_CELERY_BROKER_URL=${TWOOPSTRACKER_CELERY_BROKER_URL:-redis://redis:6379/0} env_file: - - ./twoops_tracker/.env + - ./twoops_tracker/.env.template volumes: + - app-media:/app/media - app-staticfiles:/app/staticfiles - - nginx: + + twoopstracker: image: nginx:1.20 restart: unless-stopped volumes: - ./contrib/docker-compose/nginx:/etc/nginx/conf.d + - app-media:/var/www/media - app-staticfiles:/var/www/static ports: - 8000:80 depends_on: - - twoops_tracker + - twoopstracker_app volumes: - db-data: + app-media: app-staticfiles: + db-data: From 2f24b692d71666cee48a9769505aa787eb7f5a40 Mon Sep 17 00:00:00 2001 From: _ Kilemensi Date: Mon, 10 Oct 2022 16:40:00 +0300 Subject: [PATCH 11/22] Update app name --- contrib/docker-compose/nginx/default.conf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/docker-compose/nginx/default.conf b/contrib/docker-compose/nginx/default.conf index f3e65a66..e745232b 100644 --- a/contrib/docker-compose/nginx/default.conf +++ b/contrib/docker-compose/nginx/default.conf @@ -1,5 +1,5 @@ -upstream twoops_tracker { - server twoops_tracker:8000; +upstream twoopstracker_app { + server twoopstracker_app:8000; } server { @@ -7,7 +7,7 @@ server { listen 80; location / { - proxy_pass http://twoops_tracker; + proxy_pass http://twoopstracker_app; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Host $host; proxy_redirect off; From 594e856e3e30b8fc52dedc5261af98d4449d6d1b Mon Sep 17 00:00:00 2001 From: _ Kilemensi Date: Mon, 10 Oct 2022 16:40:20 +0300 Subject: [PATCH 12/22] Add README --- dj-cmd/README.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 dj-cmd/README.md diff --git a/dj-cmd/README.md b/dj-cmd/README.md new file mode 100644 index 00000000..e89af7ef --- /dev/null +++ b/dj-cmd/README.md @@ -0,0 +1,3 @@ +## `dj-cmd` + +Command-line utility for running Django & company entry point tasks when bundled in a single `pex` binary. From 5f19821e8e849034a5ef2833758f00091dbfe3f3 Mon Sep 17 00:00:00 2001 From: _ Kilemensi Date: Mon, 10 Oct 2022 16:40:37 +0300 Subject: [PATCH 13/22] Add README --- twoops_tracker/README.md | 50 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 twoops_tracker/README.md diff --git a/twoops_tracker/README.md b/twoops_tracker/README.md new file mode 100644 index 00000000..45bbcdf8 --- /dev/null +++ b/twoops_tracker/README.md @@ -0,0 +1,50 @@ +## TwoopsTracker + +A Twitter-based disinformation tracking tool built on a web-based dashboard that collects deleted tweet content from previously identified trolls and disinformation actors. The project seeks to help monitor the social posts of known disinfo actors. The primary tangible output of the project is to expose trolls behind toxic disinformation campaigns who routinely cover their tracks by deleting original inflammatory social media posts that sparked hate speech, disinformation campaigns or conspiracy theories. + +## Getting Started + +First create `.env` file in the app directory. From project root directory, + +```sh +cp twoops_tracker/.env.template twoops_tracker/.env +``` + +and modify the `.env` file according to your needs. + +## Build + +To build a pex binary, run: + +```sh +./pants package twoops_tracker/py:twoopstracker +``` + +To build the docker image, run: + +```sh +VERSION=$(cat twoops_tracker/py/VERSION) ./pants package twoops_tracker/docker:twoopstracker +``` + +## Run + +To run pex binary, execute: + +```sh +./pants run twoops_tracker/py:twoopstracker +``` + +To run the built docker image, execute: + +```sh +docker-compose --env-file ./twoops_tracker/.env up twoopstracker +``` + +**NOTE**: You may need to run `postres` container first to make sure database +is ready to receive connections _before_ starting the `twoopstracker` app. + +To do so, run: + +```sh +docker-compose --env-file ./twoops_tracker/.env up db +``` From affc4b3bc1e87a417cdf3a7fb44dcd1d5dfe94dc Mon Sep 17 00:00:00 2001 From: _ Kilemensi Date: Mon, 10 Oct 2022 16:41:09 +0300 Subject: [PATCH 14/22] Update README to point to apps and packages --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index d4132eda..64d03b46 100644 --- a/README.md +++ b/README.md @@ -1 +1,13 @@ # Commons API + +## Applications + +| Name | Description | +| ----------------------------------------------- | -------------------------------------------- | +| [**TwoopsTracker**](./twoops_tracker/README.md) | A Twitter-based disinformation tracking tool | + +## Packages + +| Name | Description | +| ---------------------------------- | -------------------------------------------------------------- | +| [**`dj-cmd`**](./dj-cmd/README.md) | Command-line utility for running Django when bundled in `pex`. | From 90936cf8af7db8f15e3f388474f49f5eb2d08db4 Mon Sep 17 00:00:00 2001 From: _ Kilemensi Date: Mon, 10 Oct 2022 16:41:52 +0300 Subject: [PATCH 15/22] Format and lint --- dj-cmd/py/BUILD | 14 +++++++------- pants-plugins/BUILD | 2 +- .../internal_plugins/versioned_setup/register.py | 10 ++++++---- .../py/twoopstracker/authentication/BUILD | 2 +- .../py/twoopstracker/authentication/forms.py | 1 - .../authentication/migrations/0001_user_model.py | 1 - .../migrations/0002_auto_20211110_1812.py | 1 - twoops_tracker/py/twoopstracker/celery.py | 2 +- .../twoops/migrations/0001_tweet_model.py | 1 - .../twoops/migrations/0002_twitter_account.py | 1 - .../twoops/migrations/0003_userprofile.py | 1 - .../twoopstracker/twoops/migrations/0004_list.py | 1 - .../0005_add_default_values_for_fields.py | 1 - .../twoops/migrations/0006_make_fields_optional.py | 1 - .../migrations/0007_allow_fields_to_be_null.py | 1 - .../twoops/migrations/0008_tweet_deleted_at.py | 1 - .../0009_tweet_retweeted_user_screen_name.py | 1 - .../0010_rename_fields_and_add_quote_count.py | 1 - .../twoops/migrations/0010_tweetsearch.py | 1 - .../twoops/migrations/0011_alter_tweet_content.py | 1 - .../migrations/0011_alter_tweetsearch_options.py | 1 - .../0012_alter_twitteraccountslist_is_private.py | 1 - .../twoops/migrations/0013_auto_20211125_1324.py | 1 - ...14_alter_twitteraccountslist_unique_together.py | 1 - .../twoops/migrations/0015_merge_20211201_0313.py | 1 - .../twoops/migrations/0016_evidence.py | 1 - .../twoops/migrations/0017_auto_20211210_0103.py | 1 - .../twoops/migrations/0018_auto_20211214_0803.py | 1 - .../twoops/migrations/0019_alter_category_name.py | 1 - .../twoopstracker/twoops/migrations/0020_team.py | 1 - .../0021_twitteraccount_profile_image_url_https.py | 1 - .../0022_make_lists_and_teams_nullable.py | 1 - 32 files changed, 16 insertions(+), 41 deletions(-) diff --git a/dj-cmd/py/BUILD b/dj-cmd/py/BUILD index 5b831295..3fa7a4b7 100644 --- a/dj-cmd/py/BUILD +++ b/dj-cmd/py/BUILD @@ -1,6 +1,6 @@ resources( - name = "resources", - sources = [ + name="resources", + sources=[ "pyproject.toml", "setup.cfg", "README.md", @@ -9,13 +9,13 @@ resources( ) python_distribution( - name = "dj-cmd", - dependencies = [ + name="dj-cmd", + dependencies=[ "dj-cmd/py/dj_cmd:dj_cmd", ":resources", ], - provides = setup_py( - name = "dj-cmd", + provides=setup_py( + name="dj-cmd", ), - sdist = False, + sdist=False, ) diff --git a/pants-plugins/BUILD b/pants-plugins/BUILD index 8425232b..a6d76d89 100644 --- a/pants-plugins/BUILD +++ b/pants-plugins/BUILD @@ -1 +1 @@ -pants_requirements(name = "pants") +pants_requirements(name="pants") diff --git a/pants-plugins/internal_plugins/versioned_setup/register.py b/pants-plugins/internal_plugins/versioned_setup/register.py index 2d46477f..0b5cee3e 100644 --- a/pants-plugins/internal_plugins/versioned_setup/register.py +++ b/pants-plugins/internal_plugins/versioned_setup/register.py @@ -49,9 +49,9 @@ async def setup_kwargs_plugin(request: VersionedSetupKwargsRequest) -> SetupKwar version_file = kwargs.pop("version_file", None) if version and version_file: raise ValueError( - f"The python_distribution target {request.target.address} has supplied both " - "`version` and `version_file` in its setup_py's kwargs. Only one of these " - "should be supplied." + f"The python_distribution target {request.target.address} has supplied both" + " `version` and `version_file` in its setup_py's kwargs. Only one of these" + " should be supplied." ) # we default to checking VERSION file if both version and version_file are not set if not version: @@ -62,7 +62,9 @@ async def setup_kwargs_plugin(request: VersionedSetupKwargsRequest) -> SetupKwar DigestContents, PathGlobs( [version_path], - description_of_origin=f"the 'version_file' kwarg in {request.target.address}", + description_of_origin=( + f"the 'version_file' kwarg in {request.target.address}" + ), glob_match_error_behavior=GlobMatchErrorBehavior.error, ), ) diff --git a/twoops_tracker/py/twoopstracker/authentication/BUILD b/twoops_tracker/py/twoopstracker/authentication/BUILD index 771fa403..db351e28 100644 --- a/twoops_tracker/py/twoopstracker/authentication/BUILD +++ b/twoops_tracker/py/twoopstracker/authentication/BUILD @@ -1,5 +1,5 @@ python_sources( - dependencies = [ + dependencies=[ "twoops_tracker/py/twoopstracker/authentication/migrations", ], ) diff --git a/twoops_tracker/py/twoopstracker/authentication/forms.py b/twoops_tracker/py/twoopstracker/authentication/forms.py index aec8866c..bd7cf7cc 100644 --- a/twoops_tracker/py/twoopstracker/authentication/forms.py +++ b/twoops_tracker/py/twoopstracker/authentication/forms.py @@ -14,7 +14,6 @@ def save(self, request, **kwargs): token_generator = kwargs.get("token_generator", default_token_generator) for user in self.users: - uid = user_pk_to_url_str(user) token = token_generator.make_token(user) frontend_url = settings.TWOOPSTRACKER_PASSWORD_RESET_URL.rstrip("/") diff --git a/twoops_tracker/py/twoopstracker/authentication/migrations/0001_user_model.py b/twoops_tracker/py/twoopstracker/authentication/migrations/0001_user_model.py index ecc35bc1..df61541f 100644 --- a/twoops_tracker/py/twoopstracker/authentication/migrations/0001_user_model.py +++ b/twoops_tracker/py/twoopstracker/authentication/migrations/0001_user_model.py @@ -7,7 +7,6 @@ class Migration(migrations.Migration): - initial = True dependencies = [ diff --git a/twoops_tracker/py/twoopstracker/authentication/migrations/0002_auto_20211110_1812.py b/twoops_tracker/py/twoopstracker/authentication/migrations/0002_auto_20211110_1812.py index 25aab5fb..cdb4374e 100644 --- a/twoops_tracker/py/twoopstracker/authentication/migrations/0002_auto_20211110_1812.py +++ b/twoops_tracker/py/twoopstracker/authentication/migrations/0002_auto_20211110_1812.py @@ -4,7 +4,6 @@ class Migration(migrations.Migration): - dependencies = [ ("authentication", "0001_user_model"), ] diff --git a/twoops_tracker/py/twoopstracker/celery.py b/twoops_tracker/py/twoopstracker/celery.py index 03833aa8..6afffbbe 100644 --- a/twoops_tracker/py/twoopstracker/celery.py +++ b/twoops_tracker/py/twoopstracker/celery.py @@ -31,7 +31,7 @@ def start_stream_listener(): "start_stream_listener": { "task": "start_stream_listener", "schedule": crontab( - minute=f"*/{settings.TWOOPTRACKER_STREAM_LISTENER_INTERVAL}" + minute=f"*/{settings.TWOOPSTRACKER_STREAM_LISTENER_INTERVAL}" ), }, } diff --git a/twoops_tracker/py/twoopstracker/twoops/migrations/0001_tweet_model.py b/twoops_tracker/py/twoopstracker/twoops/migrations/0001_tweet_model.py index f87c81dd..d2da7297 100644 --- a/twoops_tracker/py/twoopstracker/twoops/migrations/0001_tweet_model.py +++ b/twoops_tracker/py/twoopstracker/twoops/migrations/0001_tweet_model.py @@ -5,7 +5,6 @@ class Migration(migrations.Migration): - initial = True dependencies: List[str] = [] diff --git a/twoops_tracker/py/twoopstracker/twoops/migrations/0002_twitter_account.py b/twoops_tracker/py/twoopstracker/twoops/migrations/0002_twitter_account.py index ae34b909..1ec2562a 100644 --- a/twoops_tracker/py/twoopstracker/twoops/migrations/0002_twitter_account.py +++ b/twoops_tracker/py/twoopstracker/twoops/migrations/0002_twitter_account.py @@ -5,7 +5,6 @@ class Migration(migrations.Migration): - dependencies = [ ("twoops", "0001_tweet_model"), ] diff --git a/twoops_tracker/py/twoopstracker/twoops/migrations/0003_userprofile.py b/twoops_tracker/py/twoopstracker/twoops/migrations/0003_userprofile.py index 5d295bab..b26284f5 100644 --- a/twoops_tracker/py/twoopstracker/twoops/migrations/0003_userprofile.py +++ b/twoops_tracker/py/twoopstracker/twoops/migrations/0003_userprofile.py @@ -5,7 +5,6 @@ class Migration(migrations.Migration): - dependencies = [ ("authentication", "0001_user_model"), ("twoops", "0002_twitter_account"), diff --git a/twoops_tracker/py/twoopstracker/twoops/migrations/0004_list.py b/twoops_tracker/py/twoopstracker/twoops/migrations/0004_list.py index bddd8d56..c0015b77 100644 --- a/twoops_tracker/py/twoopstracker/twoops/migrations/0004_list.py +++ b/twoops_tracker/py/twoopstracker/twoops/migrations/0004_list.py @@ -5,7 +5,6 @@ class Migration(migrations.Migration): - dependencies = [ ("twoops", "0003_userprofile"), ] diff --git a/twoops_tracker/py/twoopstracker/twoops/migrations/0005_add_default_values_for_fields.py b/twoops_tracker/py/twoopstracker/twoops/migrations/0005_add_default_values_for_fields.py index 79a56846..d4c313aa 100644 --- a/twoops_tracker/py/twoopstracker/twoops/migrations/0005_add_default_values_for_fields.py +++ b/twoops_tracker/py/twoopstracker/twoops/migrations/0005_add_default_values_for_fields.py @@ -4,7 +4,6 @@ class Migration(migrations.Migration): - dependencies = [ ("twoops", "0004_list"), ] diff --git a/twoops_tracker/py/twoopstracker/twoops/migrations/0006_make_fields_optional.py b/twoops_tracker/py/twoopstracker/twoops/migrations/0006_make_fields_optional.py index 78c139f3..9775aace 100644 --- a/twoops_tracker/py/twoopstracker/twoops/migrations/0006_make_fields_optional.py +++ b/twoops_tracker/py/twoopstracker/twoops/migrations/0006_make_fields_optional.py @@ -4,7 +4,6 @@ class Migration(migrations.Migration): - dependencies = [ ("twoops", "0005_add_default_values_for_fields"), ] diff --git a/twoops_tracker/py/twoopstracker/twoops/migrations/0007_allow_fields_to_be_null.py b/twoops_tracker/py/twoopstracker/twoops/migrations/0007_allow_fields_to_be_null.py index 70deecac..945ae3d7 100644 --- a/twoops_tracker/py/twoopstracker/twoops/migrations/0007_allow_fields_to_be_null.py +++ b/twoops_tracker/py/twoopstracker/twoops/migrations/0007_allow_fields_to_be_null.py @@ -4,7 +4,6 @@ class Migration(migrations.Migration): - dependencies = [ ("twoops", "0006_make_fields_optional"), ] diff --git a/twoops_tracker/py/twoopstracker/twoops/migrations/0008_tweet_deleted_at.py b/twoops_tracker/py/twoopstracker/twoops/migrations/0008_tweet_deleted_at.py index 94bdbdb0..ae19e1f4 100644 --- a/twoops_tracker/py/twoopstracker/twoops/migrations/0008_tweet_deleted_at.py +++ b/twoops_tracker/py/twoopstracker/twoops/migrations/0008_tweet_deleted_at.py @@ -4,7 +4,6 @@ class Migration(migrations.Migration): - dependencies = [ ("twoops", "0007_allow_fields_to_be_null"), ] diff --git a/twoops_tracker/py/twoopstracker/twoops/migrations/0009_tweet_retweeted_user_screen_name.py b/twoops_tracker/py/twoopstracker/twoops/migrations/0009_tweet_retweeted_user_screen_name.py index ff3f2cbd..a5c8b16b 100644 --- a/twoops_tracker/py/twoopstracker/twoops/migrations/0009_tweet_retweeted_user_screen_name.py +++ b/twoops_tracker/py/twoopstracker/twoops/migrations/0009_tweet_retweeted_user_screen_name.py @@ -4,7 +4,6 @@ class Migration(migrations.Migration): - dependencies = [ ("twoops", "0008_tweet_deleted_at"), ] diff --git a/twoops_tracker/py/twoopstracker/twoops/migrations/0010_rename_fields_and_add_quote_count.py b/twoops_tracker/py/twoopstracker/twoops/migrations/0010_rename_fields_and_add_quote_count.py index 53497304..a13712f9 100644 --- a/twoops_tracker/py/twoopstracker/twoops/migrations/0010_rename_fields_and_add_quote_count.py +++ b/twoops_tracker/py/twoopstracker/twoops/migrations/0010_rename_fields_and_add_quote_count.py @@ -4,7 +4,6 @@ class Migration(migrations.Migration): - dependencies = [ ("twoops", "0009_tweet_retweeted_user_screen_name"), ] diff --git a/twoops_tracker/py/twoopstracker/twoops/migrations/0010_tweetsearch.py b/twoops_tracker/py/twoopstracker/twoops/migrations/0010_tweetsearch.py index 77965eba..3ee18f94 100644 --- a/twoops_tracker/py/twoopstracker/twoops/migrations/0010_tweetsearch.py +++ b/twoops_tracker/py/twoopstracker/twoops/migrations/0010_tweetsearch.py @@ -5,7 +5,6 @@ class Migration(migrations.Migration): - dependencies = [ ("twoops", "0009_tweet_retweeted_user_screen_name"), ] diff --git a/twoops_tracker/py/twoopstracker/twoops/migrations/0011_alter_tweet_content.py b/twoops_tracker/py/twoopstracker/twoops/migrations/0011_alter_tweet_content.py index 74b8726e..60dcf091 100644 --- a/twoops_tracker/py/twoopstracker/twoops/migrations/0011_alter_tweet_content.py +++ b/twoops_tracker/py/twoopstracker/twoops/migrations/0011_alter_tweet_content.py @@ -4,7 +4,6 @@ class Migration(migrations.Migration): - dependencies = [ ("twoops", "0010_rename_fields_and_add_quote_count"), ] diff --git a/twoops_tracker/py/twoopstracker/twoops/migrations/0011_alter_tweetsearch_options.py b/twoops_tracker/py/twoopstracker/twoops/migrations/0011_alter_tweetsearch_options.py index 67322310..b763bb69 100644 --- a/twoops_tracker/py/twoopstracker/twoops/migrations/0011_alter_tweetsearch_options.py +++ b/twoops_tracker/py/twoopstracker/twoops/migrations/0011_alter_tweetsearch_options.py @@ -4,7 +4,6 @@ class Migration(migrations.Migration): - dependencies = [ ("twoops", "0010_tweetsearch"), ] diff --git a/twoops_tracker/py/twoopstracker/twoops/migrations/0012_alter_twitteraccountslist_is_private.py b/twoops_tracker/py/twoopstracker/twoops/migrations/0012_alter_twitteraccountslist_is_private.py index 59fd799f..2aa88938 100644 --- a/twoops_tracker/py/twoopstracker/twoops/migrations/0012_alter_twitteraccountslist_is_private.py +++ b/twoops_tracker/py/twoopstracker/twoops/migrations/0012_alter_twitteraccountslist_is_private.py @@ -4,7 +4,6 @@ class Migration(migrations.Migration): - dependencies = [ ("twoops", "0011_alter_tweetsearch_options"), ] diff --git a/twoops_tracker/py/twoopstracker/twoops/migrations/0013_auto_20211125_1324.py b/twoops_tracker/py/twoopstracker/twoops/migrations/0013_auto_20211125_1324.py index 2ef7be6c..334ee0ba 100644 --- a/twoops_tracker/py/twoopstracker/twoops/migrations/0013_auto_20211125_1324.py +++ b/twoops_tracker/py/twoopstracker/twoops/migrations/0013_auto_20211125_1324.py @@ -4,7 +4,6 @@ class Migration(migrations.Migration): - dependencies = [ ("twoops", "0012_alter_twitteraccountslist_is_private"), ] diff --git a/twoops_tracker/py/twoopstracker/twoops/migrations/0014_alter_twitteraccountslist_unique_together.py b/twoops_tracker/py/twoopstracker/twoops/migrations/0014_alter_twitteraccountslist_unique_together.py index fc34208b..e478ba2e 100644 --- a/twoops_tracker/py/twoopstracker/twoops/migrations/0014_alter_twitteraccountslist_unique_together.py +++ b/twoops_tracker/py/twoopstracker/twoops/migrations/0014_alter_twitteraccountslist_unique_together.py @@ -4,7 +4,6 @@ class Migration(migrations.Migration): - dependencies = [ ("twoops", "0013_auto_20211125_1324"), ] diff --git a/twoops_tracker/py/twoopstracker/twoops/migrations/0015_merge_20211201_0313.py b/twoops_tracker/py/twoopstracker/twoops/migrations/0015_merge_20211201_0313.py index 0faf50d8..f5b61d06 100644 --- a/twoops_tracker/py/twoopstracker/twoops/migrations/0015_merge_20211201_0313.py +++ b/twoops_tracker/py/twoopstracker/twoops/migrations/0015_merge_20211201_0313.py @@ -5,7 +5,6 @@ class Migration(migrations.Migration): - dependencies = [ ("twoops", "0011_alter_tweet_content"), ("twoops", "0014_alter_twitteraccountslist_unique_together"), diff --git a/twoops_tracker/py/twoopstracker/twoops/migrations/0016_evidence.py b/twoops_tracker/py/twoopstracker/twoops/migrations/0016_evidence.py index 6e3adb44..ce243590 100644 --- a/twoops_tracker/py/twoopstracker/twoops/migrations/0016_evidence.py +++ b/twoops_tracker/py/twoopstracker/twoops/migrations/0016_evidence.py @@ -5,7 +5,6 @@ class Migration(migrations.Migration): - dependencies = [ ("twoops", "0015_merge_20211201_0313"), ] diff --git a/twoops_tracker/py/twoopstracker/twoops/migrations/0017_auto_20211210_0103.py b/twoops_tracker/py/twoopstracker/twoops/migrations/0017_auto_20211210_0103.py index 1aa6bdfb..ed6db721 100644 --- a/twoops_tracker/py/twoopstracker/twoops/migrations/0017_auto_20211210_0103.py +++ b/twoops_tracker/py/twoopstracker/twoops/migrations/0017_auto_20211210_0103.py @@ -4,7 +4,6 @@ class Migration(migrations.Migration): - dependencies = [ ("twoops", "0016_evidence"), ] diff --git a/twoops_tracker/py/twoopstracker/twoops/migrations/0018_auto_20211214_0803.py b/twoops_tracker/py/twoopstracker/twoops/migrations/0018_auto_20211214_0803.py index b68a0745..c434461f 100644 --- a/twoops_tracker/py/twoopstracker/twoops/migrations/0018_auto_20211214_0803.py +++ b/twoops_tracker/py/twoopstracker/twoops/migrations/0018_auto_20211214_0803.py @@ -4,7 +4,6 @@ class Migration(migrations.Migration): - dependencies = [ ("twoops", "0017_auto_20211210_0103"), ] diff --git a/twoops_tracker/py/twoopstracker/twoops/migrations/0019_alter_category_name.py b/twoops_tracker/py/twoopstracker/twoops/migrations/0019_alter_category_name.py index f6ef4814..be25b01a 100644 --- a/twoops_tracker/py/twoopstracker/twoops/migrations/0019_alter_category_name.py +++ b/twoops_tracker/py/twoopstracker/twoops/migrations/0019_alter_category_name.py @@ -4,7 +4,6 @@ class Migration(migrations.Migration): - dependencies = [ ("twoops", "0018_auto_20211214_0803"), ] diff --git a/twoops_tracker/py/twoopstracker/twoops/migrations/0020_team.py b/twoops_tracker/py/twoopstracker/twoops/migrations/0020_team.py index be95aff8..c20648b4 100644 --- a/twoops_tracker/py/twoopstracker/twoops/migrations/0020_team.py +++ b/twoops_tracker/py/twoopstracker/twoops/migrations/0020_team.py @@ -5,7 +5,6 @@ class Migration(migrations.Migration): - dependencies = [ ("twoops", "0019_alter_category_name"), ] diff --git a/twoops_tracker/py/twoopstracker/twoops/migrations/0021_twitteraccount_profile_image_url_https.py b/twoops_tracker/py/twoopstracker/twoops/migrations/0021_twitteraccount_profile_image_url_https.py index f47afe43..b06a95be 100644 --- a/twoops_tracker/py/twoopstracker/twoops/migrations/0021_twitteraccount_profile_image_url_https.py +++ b/twoops_tracker/py/twoopstracker/twoops/migrations/0021_twitteraccount_profile_image_url_https.py @@ -4,7 +4,6 @@ class Migration(migrations.Migration): - dependencies = [ ("twoops", "0020_team"), ] diff --git a/twoops_tracker/py/twoopstracker/twoops/migrations/0022_make_lists_and_teams_nullable.py b/twoops_tracker/py/twoopstracker/twoops/migrations/0022_make_lists_and_teams_nullable.py index 8fb9516a..fa8a6e29 100644 --- a/twoops_tracker/py/twoopstracker/twoops/migrations/0022_make_lists_and_teams_nullable.py +++ b/twoops_tracker/py/twoopstracker/twoops/migrations/0022_make_lists_and_teams_nullable.py @@ -5,7 +5,6 @@ class Migration(migrations.Migration): - dependencies = [ ("twoops", "0021_twitteraccount_profile_image_url_https"), ] From de4f24203c6bd8f0da4fa014e3942796ea6c6416 Mon Sep 17 00:00:00 2001 From: _ Kilemensi Date: Mon, 10 Oct 2022 16:43:00 +0300 Subject: [PATCH 16/22] Update name --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 64d03b46..89f2f21c 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Commons API +# COMMONS API ## Applications From eb93a303264c322be45b0e6a83eb9382bb510cf5 Mon Sep 17 00:00:00 2001 From: _ Kilemensi Date: Mon, 10 Oct 2022 16:58:12 +0300 Subject: [PATCH 17/22] Update workflows to current usage --- .github/workflows/twoops-tracker-cd.yml | 2 +- .github/workflows/twoops-tracker-ci.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/twoops-tracker-cd.yml b/.github/workflows/twoops-tracker-cd.yml index 8b15cfbc..184b8d54 100644 --- a/.github/workflows/twoops-tracker-cd.yml +++ b/.github/workflows/twoops-tracker-cd.yml @@ -40,7 +40,7 @@ jobs: - name: Lint run: | - ./pants lint twoops_tracker/py:: + ./pants lint --lint-skip-formatters twoops_tracker/:: - name: Build run: | diff --git a/.github/workflows/twoops-tracker-ci.yml b/.github/workflows/twoops-tracker-ci.yml index a6244305..00b9a880 100644 --- a/.github/workflows/twoops-tracker-ci.yml +++ b/.github/workflows/twoops-tracker-ci.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.10.7", ] + python-version: ["3.10.7"] steps: - name: Cancel Previous Runs uses: styfle/cancel-workflow-action@0.10.1 @@ -28,8 +28,8 @@ jobs: - name: Lint run: | - ./pants lint twoops_tracker/py:: + ./pants lint --lint-skip-formatters twoops_tracker/:: - name: Build run: | - ./pants package twoops_tracker/py:main + ./pants package twoops_tracker/py:twoopstracker-deps twoops_tracker/py:twoopstracker-srcs From ed5c2316cc2a156cc4b8d0c2fc2dc757516cd46f Mon Sep 17 00:00:00 2001 From: Clemence Kyara Date: Tue, 11 Oct 2022 09:58:41 +0300 Subject: [PATCH 18/22] Clarify twoops_tracker/.env.template comment --- twoops_tracker/.env.template | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/twoops_tracker/.env.template b/twoops_tracker/.env.template index 1a218132..8141252f 100644 --- a/twoops_tracker/.env.template +++ b/twoops_tracker/.env.template @@ -1,14 +1,14 @@ -# This is the exhaustive list of all environemtn variables used by +# This is the exhaustive list of all environment variables used by # TwoopsTracker. If a variable is not declared here, docker-compose -# will not pass it to container at runtime. +# will not pass it to the container at runtime. # -# Do not assign values here. This *MUST* remain a just a list. -# Do not do even empty assignment e.g. +# Do not assign values here. This *MUST* remain just a list. +# Do not even do empty assignments e.g. # # TWOOPSTRACKER_DATABASE_URL= # # If you do the above, TWOOPSTRACKER_DATABASE_URL will be set to empty -# string even if a different --env-file is passed to docker-compose +# string even when a different --env-file is passed to docker-compose # Required TWOOPSTRACKER_ACCESS_TOKEN From 55d41aea6366280fdcb52ea288d1e6fa92f499c7 Mon Sep 17 00:00:00 2001 From: _ Kilemensi Date: Tue, 11 Oct 2022 10:49:01 +0300 Subject: [PATCH 19/22] Switch to simpler pants bootstrapping --- .github/workflows/twoops-tracker-cd.yml | 22 ++++++++++++++-------- .github/workflows/twoops-tracker-ci.yml | 16 +++++++++++----- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/.github/workflows/twoops-tracker-cd.yml b/.github/workflows/twoops-tracker-cd.yml index 184b8d54..7bbabc1e 100644 --- a/.github/workflows/twoops-tracker-cd.yml +++ b/.github/workflows/twoops-tracker-cd.yml @@ -4,7 +4,7 @@ on: branches: - main paths: - - 'twoops_tracker/py/VERSION' + - "twoops_tracker/py/VERSION" env: DOKKU_REMOTE_BRANCH: "master" DOKKU_REMOTE_URL: "ssh://dokku@dokku-1.dev.codeforafrica.org/twoopstracker" @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.10.7",] + python-version: ["3.10"] steps: - name: Cancel Previous Runs uses: styfle/cancel-workflow-action@0.10.1 @@ -32,17 +32,23 @@ jobs: - name: Set env run: echo "VERSION=$(cat twoops_tracker/py/VERSION)" >> $GITHUB_ENV - - name: Initialize Pants - uses: pantsbuild/actions/init-pants@main + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 with: - pants-python-version: ${{ matrix.python-version }} - pants-ci-config: "pants.toml" + python-version: ${{ matrix.python-version }} + + - name: Bootstrap Pants + run: | + ./pants --version + + - name: Check BUILD files + run: ./pants tailor --check update-build-files --check - name: Lint run: | ./pants lint --lint-skip-formatters twoops_tracker/:: - - name: Build + - name: Build Docker image run: | VERSION=${{ env.VERSION }} ./pants package twoops_tracker/docker/:: @@ -51,7 +57,7 @@ jobs: with: username: ${{ secrets.DOCKER_HUB_USERNAME }} password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - + - name: Push to DockerHub run: | docker push ${{ env.IMAGE_NAME }}:${{ env.VERSION }} diff --git a/.github/workflows/twoops-tracker-ci.yml b/.github/workflows/twoops-tracker-ci.yml index 00b9a880..7c79f353 100644 --- a/.github/workflows/twoops-tracker-ci.yml +++ b/.github/workflows/twoops-tracker-ci.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.10.7"] + python-version: ["3.10"] steps: - name: Cancel Previous Runs uses: styfle/cancel-workflow-action@0.10.1 @@ -20,11 +20,17 @@ jobs: with: fetch-depth: 0 - - name: Initialize Pants - uses: pantsbuild/actions/init-pants@main + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 with: - pants-python-version: ${{ matrix.python-version }} - pants-ci-config: "pants.toml" + python-version: ${{ matrix.python-version }} + + - name: Bootstrap Pants + run: | + ./pants --version + + - name: Check BUILD files + run: ./pants tailor --check update-build-files --check - name: Lint run: | From 4e5bb10a8376236e5348a4e7b35183616a12a777 Mon Sep 17 00:00:00 2001 From: _ Kilemensi Date: Tue, 11 Oct 2022 10:56:02 +0300 Subject: [PATCH 20/22] Fix BUILD files --- dj-cmd/py/dj_cmd/BUILD | 5 +++++ twoops_tracker/contrib/dokku/BUILD | 3 +++ twoops_tracker/docker/BUILD | 30 +++++++++++++++--------------- twoops_tracker/py/BUILD | 28 ++++++++++++++-------------- 4 files changed, 37 insertions(+), 29 deletions(-) create mode 100644 twoops_tracker/contrib/dokku/BUILD diff --git a/dj-cmd/py/dj_cmd/BUILD b/dj-cmd/py/dj_cmd/BUILD index db46e8d6..9a0e1a1a 100644 --- a/dj-cmd/py/dj_cmd/BUILD +++ b/dj-cmd/py/dj_cmd/BUILD @@ -1 +1,6 @@ python_sources() + +pex_binary( + name="cmd", + entry_point="cmd.py", +) diff --git a/twoops_tracker/contrib/dokku/BUILD b/twoops_tracker/contrib/dokku/BUILD new file mode 100644 index 00000000..3a1b556b --- /dev/null +++ b/twoops_tracker/contrib/dokku/BUILD @@ -0,0 +1,3 @@ +docker_image( + name="docker", +) diff --git a/twoops_tracker/docker/BUILD b/twoops_tracker/docker/BUILD index c20984fc..db97d822 100644 --- a/twoops_tracker/docker/BUILD +++ b/twoops_tracker/docker/BUILD @@ -1,31 +1,31 @@ docker_image( - name = "twoopstracker-deps", - image_tags = ["deps"], + name="twoopstracker-deps", + image_tags=["deps"], # We just need to hardcode something in the registries so that we can # reference it the final image's COPY instructions. Can be anything - registries = ["twoopstracker"], - repository = "app", - skip_push = True, - source = "Dockerfile.deps", + registries=["twoopstracker"], + repository="app", + skip_push=True, + source="Dockerfile.deps", ) docker_image( - name = "twoopstracker-srcs", - image_tags = ["srcs"], - registries = ["twoopstracker"], - repository = "app", - skip_push = True, - source = "Dockerfile.srcs", + name="twoopstracker-srcs", + image_tags=["srcs"], + registries=["twoopstracker"], + repository="app", + skip_push=True, + source="Dockerfile.srcs", ) docker_image( - name = "twoopstracker", - dependencies = [ + name="twoopstracker", + dependencies=[ ":twoopstracker-deps", ":twoopstracker-srcs", "twoops_tracker/sh/cmd.sh", ], - image_tags = [ + image_tags=[ "{build_args.VERSION}", "latest", ], diff --git a/twoops_tracker/py/BUILD b/twoops_tracker/py/BUILD index 49148463..f4759c66 100644 --- a/twoops_tracker/py/BUILD +++ b/twoops_tracker/py/BUILD @@ -1,6 +1,6 @@ python_sources( - name = "lib", - dependencies = [ + name="lib", + dependencies=[ "twoops_tracker/py/twoopstracker", "3rdparty/py:requirements-all#django-cors-headers", "3rdparty/py:requirements-all#django-storages", @@ -21,23 +21,23 @@ python_sources( ) pex_binary( - name = "twoopstracker-deps", - dependencies = [ + name="twoopstracker-deps", + dependencies=[ ":lib", ], - entry_point = "main.py", - include_sources = False, - include_tools = True, - layout = "packed", + entry_point="main.py", + include_sources=False, + include_tools=True, + layout="packed", ) pex_binary( - name = "twoopstracker-srcs", - dependencies = [ + name="twoopstracker-srcs", + dependencies=[ ":lib", ], - entry_point = "main.py", - include_requirements = False, - include_tools = True, - layout = "packed", + entry_point="main.py", + include_requirements=False, + include_tools=True, + layout="packed", ) From 3118c31690e5ecf6634b1f353052dfddfca63ff6 Mon Sep 17 00:00:00 2001 From: esir Date: Mon, 17 Oct 2022 15:14:39 +0300 Subject: [PATCH 21/22] Include TWOOPSTRACKER_CELERY_BROKER_URL in the .env.template file --- twoops_tracker/.env.template | 1 + 1 file changed, 1 insertion(+) diff --git a/twoops_tracker/.env.template b/twoops_tracker/.env.template index 8141252f..1d114f12 100644 --- a/twoops_tracker/.env.template +++ b/twoops_tracker/.env.template @@ -13,6 +13,7 @@ # Required TWOOPSTRACKER_ACCESS_TOKEN TWOOPSTRACKER_ACCESS_TOKEN_SECRET +TWOOPSTRACKER_CELERY_BROKER_URL TWOOPSTRACKER_CONSUMER_KEY TWOOPSTRACKER_CONSUMER_SECRET TWOOPSTRACKER_DATABASE_URL From b11a8bedd0cab8e77a7e58f93547bd5578114aea Mon Sep 17 00:00:00 2001 From: esir Date: Mon, 17 Oct 2022 15:24:22 +0300 Subject: [PATCH 22/22] Bump up twoopstracker to version 2.0.0a6 --- twoops_tracker/contrib/dokku/Dockerfile | 2 +- twoops_tracker/py/VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/twoops_tracker/contrib/dokku/Dockerfile b/twoops_tracker/contrib/dokku/Dockerfile index e9845e31..9c9f2168 100644 --- a/twoops_tracker/contrib/dokku/Dockerfile +++ b/twoops_tracker/contrib/dokku/Dockerfile @@ -1 +1 @@ -FROM codeforafrica/twoopstracker:2.0.0a5 +FROM codeforafrica/twoopstracker:2.0.0a6 diff --git a/twoops_tracker/py/VERSION b/twoops_tracker/py/VERSION index 0691e44f..758664df 100644 --- a/twoops_tracker/py/VERSION +++ b/twoops_tracker/py/VERSION @@ -1 +1 @@ -2.0.0a5 +2.0.0a6