Converts kizuna to worker based model

Use slack events API to have events pushed to kizuna rather than pulled
from rtm api

api process serves a lightweight json api that enqueues messages on
rabbitmq

worker process uses dramatiq actors to async take care of messages

Dockerfile.api

@@ -0,0 +1,5 @@
+FROM austinpray/kizuna/base
+
+CMD ["gunicorn", "--config", "python:config.gunicorn_api", "api:app"]
+
+COPY . ${workdir}

Dockerfile.base

@@ -8,8 +8,10 @@ ENV PYTHONPATH="${PYTHONPATH}:${workdir}"
 RUN apt-get update \
     && apt-get install -y vim graphviz
 
-COPY requirements.txt ${workdir}/requirements.txt
-
-RUN pip --no-cache-dir install -r requirements.txt
-
+# heavy deps
+RUN pip install spacy
 RUN python -m spacy download en
+
+# regular deps
+COPY requirements.txt ${workdir}/requirements.txt
+RUN pip install -r requirements.txt

Dockerfile.web

@@ -1,5 +1,5 @@
 FROM austinpray/kizuna/base
 
-CMD ["gunicorn", "--config", "python:config.gunicorn", "web:app"]
+CMD ["gunicorn", "--config", "python:config.gunicorn_web", "web:app"]
 
 COPY . ${workdir}

Dockerfile.bot → Dockerfile.worker

@@ -1,5 +1,5 @@
 FROM austinpray/kizuna/base
 
-CMD ["python", "-u", "./bot.py"]
+CMD ["dramatiq", "worker"]
 
 COPY . ${workdir}

Makefile

@@ -1,81 +1,125 @@
-.PHONY: kub_deploy pep8 registry_push web bot build pull perm dev_info base migrate_dev
+.PHONY: kub_deploy pep8 registry_push web api build pull perm dev_info base migrate_dev
 
+# simulate CI environment
+TRAVIS_COMMIT ?= $(shell git rev-parse HEAD)
+
+# tags used locally
 base_tag = austinpray/kizuna/base
-bot_tag = austinpray/kizuna/bot
+
+api_tag = austinpray/kizuna/api
 web_tag = austinpray/kizuna/web
+worker_tag = austinpray/kizuna/worker
 
-registry_base_url = registry.heroku.com/kizunaai
-registry_base_url = us.gcr.io/kizuna-188702
+# remote registry prefix for pushing to gcloud
+registry_prefix = us.gcr.io/kizuna-188702
 
-TRAVIS_COMMIT ?= $(shell git rev-parse HEAD)
+# base tags for different images in project
+registry_base_tag = $(registry_prefix)/base
 
-registry_base_tag = $(registry_base_url)/base
-registry_bot_tag = $(registry_base_url)/bot
-registry_web_tag = $(registry_base_url)/web
+registry_api_tag = $(registry_prefix)/api
+registry_web_tag = $(registry_prefix)/web
+registry_worker_tag = $(registry_prefix)/worker
 
+# commit level tag
 registry_base_tag_commit = $(registry_base_tag):$(TRAVIS_COMMIT)
-registry_bot_tag_commit = $(registry_bot_tag):$(TRAVIS_COMMIT)
+
+registry_api_tag_commit = $(registry_api_tag):$(TRAVIS_COMMIT)
 registry_web_tag_commit = $(registry_web_tag):$(TRAVIS_COMMIT)
+registry_worker_tag_commit = $(registry_worker_tag):$(TRAVIS_COMMIT)
 
+# latest tags
 registry_base_tag_latest = $(registry_base_tag):latest
-registry_bot_tag_latest = $(registry_bot_tag):latest
+
+registry_api_tag_latest = $(registry_api_tag):latest
 registry_web_tag_latest = $(registry_web_tag):latest
+registry_worker_tag_latest = $(registry_worker_tag):latest
 
-build: dev_info bot web
+# build everything
+build: dev_info api worker web
 
+# check the project for pep8 compliance
 pep8:
 	docker run --rm -v $(shell pwd):/code omercnet/pycodestyle --show-source /code
 
+# make a dev-info file so kizuna knows what commit she's on
+dev_info:
+	bin/generate-dev-info.py --revision $(TRAVIS_COMMIT) > .dev-info.json
+
+# push all the images to gcloud registry
 registry_push:
 	docker tag $(base_tag) $(registry_base_tag_commit)
-	docker tag $(bot_tag) $(registry_bot_tag_commit)
+	docker tag $(api_tag) $(registry_api_tag_commit)
 	docker tag $(web_tag) $(registry_web_tag_commit)
+	docker tag $(worker_tag) $(registry_worker_tag_commit)
 	gcloud docker -- push $(registry_base_tag_commit)
-	gcloud docker -- push $(registry_bot_tag_commit)
+	gcloud docker -- push $(registry_api_tag_commit)
 	gcloud docker -- push $(registry_web_tag_commit)
-
+	gcloud docker -- push $(registry_worker_tag_commit)
+	docker tag $(base_tag) $(registry_base_tag_latest)
+	docker tag $(api_tag) $(registry_api_tag_latest)
+	docker tag $(web_tag) $(registry_web_tag_latest)
+	docker tag $(worker_tag) $(registry_worker_tag_latest)
+	gcloud docker -- push $(registry_base_tag_latest)
+	gcloud docker -- push $(registry_api_tag_latest)
+	gcloud docker -- push $(registry_web_tag_latest)
+	gcloud docker -- push $(registry_worker_tag_latest)
+
+# release the current commit to kube
 kube_deploy: registry_push
-	kubectl set image deployment/bot bot=$(registry_bot_tag_commit)
+	kubectl set image deployment/api api=$(registry_api_tag_commit)
 	kubectl set image deployment/web web=$(registry_web_tag_commit)
+	kubectl set image deployment/worker worker=$(registry_worker_tag_commit)
 
+# pull images from registry prolly for caching reasons
 pull:
 	gcloud docker -- pull $(registry_base_tag)
-	gcloud docker -- pull $(registry_bot_tag)
+	gcloud docker -- pull $(registry_api_tag)
 	gcloud docker -- pull $(registry_web_tag)
+	gcloud docker -- pull $(registry_worker_tag)
 	docker tag $(registry_base_tag) $(base_tag)
-	docker tag $(registry_bot_tag) $(bot_tag)
+	docker tag $(registry_api_tag) $(api_tag)
 	docker tag $(registry_web_tag) $(web_tag)
+	docker tag $(registry_worker_tag) $(worker_tag)
 
+# image building
 base:
 	docker build \
 		--file Dockerfile.base \
 		--cache-from $(base_tag) \
 		-t $(base_tag) \
 		.
 
-bot: base
+api: base
 	docker build \
-		--file Dockerfile.bot \
-		--cache-from $(bot_tag) \
-		-t $(bot_tag) \
+		--file Dockerfile.api \
+		--cache-from $(api_tag) \
+		-t $(api_tag) \
 		.
 
-web: bot
+web: api
 	docker run -it --rm --name build-web-assets -v $(shell pwd):/kizuna -w /kizuna node:9 npm run build
 	docker build \
 		--file Dockerfile.web \
 		--cache-from $(web_tag) \
 		-t $(web_tag) \
 		.
 
+worker: base
+	docker build \
+		--file Dockerfile.worker \
+		--cache-from $(worker_tag) \
+		-t $(worker_tag) \
+		.
+
+# docker permissions helper
 perm:
 	sudo chown -R $(shell whoami):$(shell whoami) .
 
-dev_info:
-	bin/generate-dev-info.py --revision $(TRAVIS_COMMIT) > .dev-info.json
-
+# dev commands
+## watch for file changes and restart accordingly
 dev:
-	nodemon -e 'py' --exec docker-compose restart bot web
+	nodemon -e 'py' --exec docker-compose restart api web
 
+## run dev migrations
 migrate_dev:
-	docker-compose run bot alembic upgrade head
+	docker-compose run api alembic upgrade head

api.py

@@ -0,0 +1,55 @@
+import falcon
+import json
+import config
+import logging
+
+from worker import worker
+
+
+class HealthCheckResource(object):
+    def on_get(self, req, resp):
+        resp.body = '{"ok": true}'
+
+
+class EventsResource(object):
+
+    def __init__(self):
+        self.logger = logging.getLogger('thingsapp.' + __name__)
+
+    def on_post(self, req, resp):
+        if not req.content_length:
+            resp.status = falcon.HTTP_400
+            resp.body = 'go away'
+            return
+
+        doc = json.load(req.stream)
+        print(doc)
+
+        callback_type = doc['type']
+
+        if doc['token'] != config.SLACK_VERIFICATION_TOKEN:
+            resp.status = falcon.HTTP_401
+            resp.body = 'go away'
+            return
+
+        if callback_type == 'url_verification':
+            resp.body = doc['challenge']
+            return
+
+        if callback_type == 'event_callback':
+            event = doc['event']
+            event_type = event['type']
+            if event_type == 'message':
+                self.logger.debug(event)
+                worker.send(event)
+
+        resp.body = 'thanks!'
+
+
+app = falcon.API()
+
+events = EventsResource()
+app.add_route('/slack/events', events)
+
+healthChecks = HealthCheckResource()
+app.add_route('/', healthChecks)

config/__init__.py

@@ -14,9 +14,10 @@
 SECRET_KEY = os.environ.get('SECRET_KEY', None)
 SENTRY_URL = os.environ.get('SENTRY_URL', None)
 SLACK_API_TOKEN = os.environ.get('SLACK_API_TOKEN', None)
+SLACK_VERIFICATION_TOKEN = os.environ.get('SLACK_VERIFICATION_TOKEN', None)
 
 FERNET_KEY = os.environ.get('FERNET_KEY', None)
 if FERNET_KEY:
     FERNET_KEY = FERNET_KEY.encode('ascii')
 
-FERNET_TTL = DAY_IN_SECONDS if KIZUNA_ENV != 'development' else 30*DAY_IN_SECONDS
+FERNET_TTL = DAY_IN_SECONDS if KIZUNA_ENV != 'development' else 30 * DAY_IN_SECONDS

config/gunicorn_api.py

@@ -0,0 +1,7 @@
+from . import KIZUNA_ENV
+
+if KIZUNA_ENV == 'development':
+    bind = '0.0.0.0:8001'
+    loglevel = 'debug'
+
+workers = 4