Use redis as backend for celery. Also re-enable autoscale (#22)

Signed-off-by: Alex Co <alex.tuan@mindvalley.com>

backend/danswer/background/celery/celery.py

@@ -37,9 +37,8 @@
 logger = setup_logger()
 
 connection_string = build_connection_string(db_api=SYNC_DB_API)
-# celery_broker_url = f"sqla+{connection_string}"
 celery_broker_url = "redis://redis-server-service:6379/0"
-celery_backend_url = f"db+{connection_string}"
+celery_backend_url = "redis://redis-server-service:6379/0"
 celery_app = Celery(__name__, broker=celery_broker_url, backend=celery_backend_url)
 
 

backend/supervisord.conf

@@ -25,7 +25,7 @@ autorestart=true
 # relatively compute-light (e.g. they tend to just make a bunch of requests to 
 # Vespa / Postgres)
 [program:celery_worker]
-command=celery -A danswer.background.celery worker --pool=threads --loglevel=DEBUG --logfile=/var/log/celery_worker.log
+command=celery -A danswer.background.celery worker --pool=threads --autoscale=10,3 --loglevel=DEBUG --logfile=/var/log/celery_worker.log
 stdout_logfile=/var/log/celery_worker_supervisor.log
 stdout_logfile_maxbytes=52428800
 redirect_stderr=true