From 3f7add7d628e5cbf30a51105f14b2a0dc20157d5 Mon Sep 17 00:00:00 2001 From: Mirian Okradze Date: Thu, 2 Nov 2023 16:40:05 +0400 Subject: [PATCH] feat: move fine tuning in worker, add fine tuning endpoints for worker --- apps/server/controllers/configuration.py | 2 +- apps/server/controllers/fine_tuning.py | 26 ++++- apps/server/models/config.py | 6 +- apps/server/models/fine_tuning.py | 45 ++++++-- apps/server/services/chat.py | 2 +- apps/server/services/fine_tuning.py | 51 ++++----- apps/worker/docker-compose.yml | 18 +++ apps/worker/helpers.py | 28 ----- apps/worker/main.py | 58 +++++++++- apps/worker/poetry.lock | 139 ++++++++++++++++++++++- apps/worker/pyproject.toml | 1 + 11 files changed, 295 insertions(+), 81 deletions(-) create mode 100644 apps/worker/docker-compose.yml delete mode 100644 apps/worker/helpers.py diff --git a/apps/server/controllers/configuration.py b/apps/server/controllers/configuration.py index 1ac33e593..8e92ce996 100644 --- a/apps/server/controllers/configuration.py +++ b/apps/server/controllers/configuration.py @@ -28,7 +28,7 @@ def index_documents(value: str, datasource_id: UUID, account: AccountOutput): session = create_session() - settings = ConfigModel.get_account_settings(session, account) + settings = ConfigModel.get_account_settings(session, account.id) datasource = DatasourceModel.get_datasource_by_id(session, datasource_id, account) try: diff --git a/apps/server/controllers/fine_tuning.py b/apps/server/controllers/fine_tuning.py index 1b2be4ba7..60446404d 100644 --- a/apps/server/controllers/fine_tuning.py +++ b/apps/server/controllers/fine_tuning.py @@ -7,7 +7,7 @@ from exceptions import FineTuningNotFoundException from models.config import ConfigModel from models.fine_tuning import FineTuningModel -from services.fine_tuning import fine_tune_openai_model +from services.fine_tuning import check_fine_tuning, fine_tune_openai_model from typings.auth import UserAccount from typings.fine_tuning import FineTuningInput, FineTuningOutput from utils.auth import authenticate @@ -17,6 +17,14 @@ router = APIRouter() +@router.post( + "/{fine_tuning_id}/check", status_code=200, response_model=FineTuningOutput +) +def check_fine_tuning_status(fine_tuning_id: UUID): + check_fine_tuning(db.session, fine_tuning_id) + return {"message": "Fine-tuning status checked"} + + @router.post("", status_code=201, response_model=FineTuningOutput) def create_fine_tuning( fine_tuning: FineTuningInput, @@ -30,7 +38,7 @@ def create_fine_tuning( auth.account.id, ) - settings = ConfigModel.get_account_settings(db.session, auth.account) + settings = ConfigModel.get_account_settings(db.session, auth.account.id) if not settings.openai_api_key: raise HTTPException( @@ -85,6 +93,20 @@ def update_fine_tuning( raise HTTPException(status_code=404, detail="Fine-tuning not found") +@router.get("/pending", response_model=List[FineTuningOutput]) +def get_pending_fine_tunings( + auth: UserAccount = Depends(authenticate), +) -> List[FineTuningOutput]: + """ + Get all pending fine-tunings for worker. + + Returns: + List[FineTuningOutput]: List of pending fine-tunings associated. + """ + fine_tuning_models = FineTuningModel.get_pending_fine_tunings(db.session) + return convert_fine_tunings_to_fine_tuning_list(fine_tuning_models) + + @router.get("/{id}", response_model=FineTuningOutput) def get_fine_tuning_by_id( id: UUID, auth: UserAccount = Depends(authenticate) diff --git a/apps/server/models/config.py b/apps/server/models/config.py index 10eed7484..8de4372bf 100644 --- a/apps/server/models/config.py +++ b/apps/server/models/config.py @@ -247,7 +247,9 @@ def get_config_by_session_id( return config @classmethod - def get_account_settings(cls, session: Session, account) -> AccountSettings: + def get_account_settings( + cls, session: Session, account_id: UUID + ) -> AccountSettings: keys = [ "open_api_key", "hugging_face_access_token", @@ -262,7 +264,7 @@ def get_account_settings(cls, session: Session, account) -> AccountSettings: session.query(ConfigModel) .filter( ConfigModel.key.in_(keys), - ConfigModel.account_id == account.id, + ConfigModel.account_id == account_id, or_( or_( ConfigModel.is_deleted.is_(False), diff --git a/apps/server/models/fine_tuning.py b/apps/server/models/fine_tuning.py index 222bec37c..01fbc629d 100644 --- a/apps/server/models/fine_tuning.py +++ b/apps/server/models/fine_tuning.py @@ -1,6 +1,7 @@ from __future__ import annotations import uuid +from typing import Optional from sqlalchemy import UUID, Boolean, Column, ForeignKey, String from sqlalchemy.orm import Session, relationship @@ -160,26 +161,46 @@ def get_fine_tunings(cls, session: Session, account_id: UUID): ) @classmethod - def get_fine_tuning_by_id(cls, session: Session, id: UUID, account_id: UUID): + def get_pending_fine_tunings(cls, session: Session): + return ( + session.query(FineTuningModel) + .filter( + FineTuningModel.status.in_( + [ + FineTuningStatus.VALIDATING.value, + FineTuningStatus.QUEUED.value, + FineTuningStatus.RUNNING.value, + ] + ), + FineTuningModel.is_deleted.is_(False), + ) + .all() + ) + + @classmethod + def get_fine_tuning_by_id( + cls, session: Session, id: UUID, account_id: Optional[UUID] = None + ): """ - Get Datasource from datasource_id + Get FineTuningModel from id Args: session: The database session. - datasource_id(int) : Unique identifier of an Datasource. + id(UUID) : Unique identifier of a FineTuningModel. + account_id(UUID, optional) : Unique identifier of an account. Defaults to None. Returns: - Datasource: Datasource object is returned. + FineTuningModel: FineTuningModel object is returned. """ - fine_tuning_model = ( - session.query(FineTuningModel) - .filter( - FineTuningModel.id == id, - FineTuningModel.account_id == account_id, - FineTuningModel.is_deleted.is_(False), - ) - .first() + query = session.query(FineTuningModel).filter( + FineTuningModel.id == id, + FineTuningModel.is_deleted.is_(False), ) + + if account_id is not None: + query = query.filter(FineTuningModel.account_id == account_id) + + fine_tuning_model = query.first() return fine_tuning_model @classmethod diff --git a/apps/server/services/chat.py b/apps/server/services/chat.py index 1aa8ae041..d5761ad7b 100644 --- a/apps/server/services/chat.py +++ b/apps/server/services/chat.py @@ -194,7 +194,7 @@ def process_chat_message( run_id=run.id, ) - settings = ConfigModel.get_account_settings(db.session, provider_account) + settings = ConfigModel.get_account_settings(db.session, provider_account.id) if len(agents) > 0: for agent_with_configs in agents: diff --git a/apps/server/services/fine_tuning.py b/apps/server/services/fine_tuning.py index a79229467..4aa7fcc54 100644 --- a/apps/server/services/fine_tuning.py +++ b/apps/server/services/fine_tuning.py @@ -8,7 +8,9 @@ import openai from openai.error import AuthenticationError +from sqlalchemy.orm import Session +from models.config import ConfigModel from models.fine_tuning import FineTuningModel from services.aws_s3 import AWSS3Service from typings.config import AccountSettings @@ -66,43 +68,36 @@ def fine_tune_openai_model( ) fine_tuning_model.openai_fine_tuning_id = fine_tuning_job.id + except AuthenticationError: + fine_tuning_model.error = "Invalid OpenAI API Key" + except Exception as err: + fine_tuning_model.error = str(err) - def retrieve_job(): - job = openai.FineTuningJob.retrieve( - api_key=settings.openai_api_key, id=fine_tuning_job.id - ) - - session.refresh(fine_tuning_model) - - fine_tuning_model.status = OPENAI_TO_FINE_TUNING_STATUS[job.status].value + session.commit() - is_finished = False - if fine_tuning_model.status == FineTuningStatus.COMPLETED.value: - fine_tuning_model.model_identifier = job.fine_tuned_model - is_finished = True +def check_fine_tuning(session: Session, id: UUID): + fine_tuning_model = FineTuningModel.get_fine_tuning_by_id(session, id) + settings = ConfigModel.get_account_settings(session, fine_tuning_model.account_id) - if job.error: - fine_tuning_model.error = job.error - is_finished = True + fine_tuning_job = openai.FineTuningJob.retrieve( + id=fine_tuning_model.openai_fine_tuning_id, + api_key=settings.openai_api_key, + ) - session.commit() + job = openai.FineTuningJob.retrieve( + api_key=settings.openai_api_key, id=fine_tuning_job.id + ) - return is_finished + fine_tuning_model.status = OPENAI_TO_FINE_TUNING_STATUS[job.status].value - while True: - is_finished = retrieve_job() + if fine_tuning_model.status == FineTuningStatus.COMPLETED.value: + fine_tuning_model.model_identifier = job.fine_tuned_model - if is_finished: - break + if job.error: + fine_tuning_model.error = job.error - time.sleep(60) - except AuthenticationError: - fine_tuning_model.error = "Invalid OpenAI API Key" - session.commit() - except Exception as err: - fine_tuning_model.error = str(err) - session.commit() + session.commit() def convert_message_to_openai_conversation_format(message: Dict): diff --git a/apps/worker/docker-compose.yml b/apps/worker/docker-compose.yml new file mode 100644 index 000000000..ef9202416 --- /dev/null +++ b/apps/worker/docker-compose.yml @@ -0,0 +1,18 @@ +version: "3.8" + +services: + redis: + image: redis:latest + ports: + - "6379:6379" + + worker: + build: + context: . + dockerfile: docker/Dockerfile + volumes: + - .:/app + ports: + - "3001:80" + depends_on: + - redis diff --git a/apps/worker/helpers.py b/apps/worker/helpers.py deleted file mode 100644 index f50c4dcb6..000000000 --- a/apps/worker/helpers.py +++ /dev/null @@ -1,28 +0,0 @@ -import asyncio -import httpx -from config import Config - - -async def fetch_scheduled_runs(): - async with httpx.AsyncClient() as client: - res = await client.get(f"{Config.SERVER_URL}/schedule/due") - - return res.json() - - -async def run_schedule(schedule_id: str): - async with httpx.AsyncClient() as client: - res = await client.post(f"{Config.SERVER_URL}/schedule/{schedule_id}/run") - - return res - - -async def execute_scheduled_runs(): - schedules_with_configs = await fetch_scheduled_runs() - - await asyncio.gather( - *( - run_schedule(schedule["schedule"]["id"]) - for schedule in schedules_with_configs - ) - ) diff --git a/apps/worker/main.py b/apps/worker/main.py index 51bc62b18..2afd24eb9 100644 --- a/apps/worker/main.py +++ b/apps/worker/main.py @@ -1,10 +1,11 @@ import asyncio from datetime import timedelta +import httpx +import requests from celery import Celery from config import Config -from helpers import execute_scheduled_runs app = Celery("l3agi", include=["main"], imports=["main"]) @@ -14,10 +15,14 @@ app.conf.accept_content = ["application/x-python-serialize", "application/json"] CELERY_BEAT_SCHEDULE = { - "execute-scheduled-runs": { - "task": "execute-scheduled-runs", + "register-scheduled-run-tasks": { + "task": "register-scheduled-run-tasks", "schedule": timedelta(minutes=2), }, + "register-fine-tuning-tasks": { + "task": "register-fine-tuning-tasks", + "schedule": timedelta(minutes=5), + }, } @@ -25,15 +30,56 @@ @app.task( - name="execute-scheduled-runs", + name="register-scheduled-run-tasks", autoretry_for=(Exception,), retry_backoff=2, max_retries=5, ) def execute_scheduled_runs_task(): - print("Running scheduled agents") + res = requests.get(f"{Config.SERVER_URL}/schedule/due") + schedules_with_configs = res.json() + + for schedule in schedules_with_configs: + execute_single_schedule_task.apply_async(args=[schedule["schedule"]["id"]]) + + schedule_ids = [schedule["schedule"]["id"] for schedule in schedules_with_configs] + return schedule_ids + + +@app.task( + name="execute-single-schedule", + autoretry_for=(Exception,), + retry_backoff=2, + max_retries=5, +) +def execute_single_schedule_task(schedule_id: str): + res = requests.post(f"{Config.SERVER_URL}/schedule/{schedule_id}/run") + return res.json() + + +@app.task( + name="register-fine-tuning-tasks", + autoretry_for=(Exception,), + retry_backoff=2, + max_retries=5, +) +def register_fine_tunings_task(): + res = requests.get(f"{Config.SERVER_URL}/fine-tuning/pending") + fine_tunings = res.json() + + for fine_tuning in fine_tunings: + check_single_fine_tuning_task.apply_async(args=[fine_tuning["id"]]) - asyncio.run(execute_scheduled_runs()) + +@app.task( + name="check-single-fine-tuning", + autoretry_for=(Exception,), + retry_backoff=2, + max_retries=5, +) +def check_single_fine_tuning_task(fine_tuning_id: str): + res = requests.post(f"{Config.SERVER_URL}/fine-tuning/{fine_tuning_id}/check") + return res if __name__ == "__main__": diff --git a/apps/worker/poetry.lock b/apps/worker/poetry.lock index bcda5ea0e..17f071ba4 100644 --- a/apps/worker/poetry.lock +++ b/apps/worker/poetry.lock @@ -122,6 +122,105 @@ files = [ {file = "certifi-2023.7.22.tar.gz", hash = "sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082"}, ] +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + [[package]] name = "click" version = "8.1.7" @@ -411,6 +510,27 @@ async-timeout = {version = ">=4.0.2", markers = "python_full_version <= \"3.11.2 hiredis = ["hiredis (>=1.0.0)"] ocsp = ["cryptography (>=36.0.1)", "pyopenssl (==20.0.1)", "requests (>=2.26.0)"] +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + [[package]] name = "six" version = "1.16.0" @@ -464,6 +584,23 @@ files = [ {file = "tzdata-2023.3.tar.gz", hash = "sha256:11ef1e08e54acb0d4f95bdb1be05da659673de4acbd21bf9c69e94cc5e907a3a"}, ] +[[package]] +name = "urllib3" +version = "2.0.7" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.7" +files = [ + {file = "urllib3-2.0.7-py3-none-any.whl", hash = "sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e"}, + {file = "urllib3-2.0.7.tar.gz", hash = "sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +secure = ["certifi", "cryptography (>=1.9)", "idna (>=2.0.0)", "pyopenssl (>=17.1.0)", "urllib3-secure-extra"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + [[package]] name = "vine" version = "5.0.0" @@ -489,4 +626,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "eabc647396caa959b890daea98768f81d2dc851f940ddba7ea7a694f956772f5" +content-hash = "6fadae3cb15216b15d9abc75cbce954bc54bba3620e9607f14753db1f2cf9f9a" diff --git a/apps/worker/pyproject.toml b/apps/worker/pyproject.toml index c8935c276..b8a2d9094 100644 --- a/apps/worker/pyproject.toml +++ b/apps/worker/pyproject.toml @@ -12,6 +12,7 @@ python-dotenv = "^1.0.0" redis = "^5.0.1" httpx = "^0.25.0" flower = "^2.0.1" +requests = "^2.31.0" [build-system]