Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

(BSR)[API] feat: extract import_deposit_csv command logic for reuse p… #15151

Merged
merged 1 commit into from
Nov 21, 2024
Merged

(BSR)[API] feat: extract import_deposit_csv command logic for reuse p… #15151

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 80 additions & 9 deletions api/src/pcapi/core/educational/api/institution.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
import csv
from datetime import datetime
from decimal import Decimal
import logging
import os

from flask_sqlalchemy import BaseQuery
import sqlalchemy as sa
Expand All @@ -20,6 +23,9 @@
import pcapi.utils.postal_code as postal_code_utils


logger = logging.getLogger(__name__)


def get_all_educational_institutions(page: int, per_page_limit: int) -> tuple[tuple, int]:
offset = (per_page_limit * (page - 1)) if page > 0 else 0
return educational_repository.get_all_educational_institutions(offset=offset, limit=per_page_limit)
Expand Down Expand Up @@ -53,6 +59,60 @@ def search_educational_institution(
)


def import_deposit_institution_csv(
*, path: str, year: int, ministry: str, conflict: str, final: bool, commit: bool
) -> Decimal:
"""
Import deposits from csv file and update institutions according to adage data
Return the total imported amount
"""

if not os.path.exists(path):
raise ValueError("The given file does not exist")

try:
educational_year = educational_repository.get_educational_year_beginning_at_given_year(year)
except educational_exceptions.EducationalYearNotFound:
raise ValueError(f"Educational year not found for year {year}")

with open(path, "r", encoding="utf-8") as csv_file:
csv_rows = csv.DictReader(csv_file, delimiter=";")
headers = csv_rows.fieldnames
if not headers or ("UAICode" not in headers and "UAI" not in headers):
raise ValueError("UAICode or depositAmount missing in CSV headers")

data: dict[str, Decimal] = {}
# sometimes we get 1 row per institution and sometimes 1 row per class.
for row in csv_rows:
# try to get the UAI
uai_header = "UAI" if "UAI" in headers else "UAICode"
uai = row[uai_header].strip()
# try to get the amount
if "Crédits de dépenses" in headers or "depositAmount" in headers:
amount_header = "depositAmount" if "depositAmount" in headers else "Crédits de dépenses"
amount = Decimal(row[amount_header])
elif "montant par élève" in headers and "Effectif" in headers:
amount = Decimal(row["Effectif"]) * Decimal(row["montant par élève"])
else:
raise ValueError("Now way to get the amount found")

if uai in data:
data[uai] += amount
else:
data[uai] = amount

logger.info("Finished reading data from csv, starting deposit import")
total_amount = import_deposit_institution_data(
data=data,
educational_year=educational_year,
ministry=educational_models.Ministry[ministry],
conflict=conflict,
final=final,
commit=commit,
)
return total_amount


def import_deposit_institution_data(
*,
data: dict[str, Decimal],
Expand All @@ -61,20 +121,22 @@ def import_deposit_institution_data(
final: bool,
conflict: str,
commit: bool,
) -> None:
) -> Decimal:
adage_institutions = {
i.uai: i for i in adage_client.get_adage_educational_institutions(ansco=educational_year.adageId)
}
db_institutions = {
institution.institutionId: institution for institution in educational_models.EducationalInstitution.query.all()
}

not_found_uais = [uai for uai in data if uai not in adage_institutions]
if not_found_uais:
raise ValueError(f"UAIs not found in adage: {not_found_uais}")

total_amount = Decimal(0)
for uai, amount in data.items():
created = False
adage_institution = adage_institutions.get(uai)
if not adage_institution:
print(f"\033[91mERROR: UAI:{uai} not found in adage.\033[0m")
return

adage_institution = adage_institutions[uai]
db_institution = db_institutions.get(uai, None)
institution_type = INSTITUTION_TYPES.get(adage_institution.sigle, adage_institution.sigle)
if db_institution:
Expand All @@ -87,7 +149,7 @@ def import_deposit_institution_data(
db_institution.isActive = True
else:
created = True
print(f"\033[33mWARNING: UAI:{uai} not found in db, creating institution.\033[0m")
logger.warning("UAI:%s not found in db, creating institution", uai)
db_institution = educational_models.EducationalInstitution(
institutionId=uai,
institutionType=institution_type,
Expand All @@ -109,8 +171,11 @@ def import_deposit_institution_data(

if deposit:
if deposit.ministry != ministry and conflict == "replace":
print(
f"\033[33mWARNING: Ministry changed from '{deposit.ministry.name}' to '{ministry.name}' for deposit {deposit.id}.\033[0m"
logger.warning(
"Ministry changed from '%s' to '%s' for deposit %s",
deposit.ministry.name,
ministry.name,
deposit.id,
)
deposit.ministry = ministry
deposit.amount = amount
Expand All @@ -125,8 +190,14 @@ def import_deposit_institution_data(
)
db.session.add(deposit)

total_amount += amount

if commit:
db.session.commit()
else:
db.session.flush()
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Je pense qu'un log final avec notamment les montant importés facilitera validation/debug

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Genre la somme des amount ? Dans la doc du job console on a :

Il est interdit de print ou logger.* données à caractère personnel, pour cela [utiliser des outputs via [OUTPUT_DIRECTORY]

Je sais pas si c'est une donnée "sensible"

Les autres logs sont sur des UAIs qui sont publics, ça m'a paru OK


return total_amount


def get_current_year_remaining_credit(institution: educational_models.EducationalInstitution) -> Decimal:
Expand Down
50 changes: 3 additions & 47 deletions api/src/pcapi/core/educational/commands.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,14 +1,11 @@
import csv
import datetime
from decimal import Decimal
import logging
import os

import click

from pcapi import settings
from pcapi.core import search
from pcapi.core.educational import exceptions as educational_exceptions
from pcapi.core.educational import repository as educational_repository
from pcapi.core.educational.api import booking as educational_api_booking
import pcapi.core.educational.api.adage as adage_api
Expand Down Expand Up @@ -75,50 +72,9 @@ def import_deposit_csv(*, path: str, year: int, ministry: str, conflict: str, fi

CSV format change every time we try to work with it.
"""
if not os.path.exists(path):
print("\033[91mERROR: The given file does not exists.\033[0m")
return

try:
educational_year = educational_repository.get_educational_year_beginning_at_given_year(year)
except educational_exceptions.EducationalYearNotFound:
print(f"\033[91mERROR: Educational year not found for year {year}.\033[0m")
return
with open(path, "r", encoding="utf-8") as csv_file:
csv_rows = csv.DictReader(csv_file, delimiter=";")
headers = csv_rows.fieldnames
if not headers or ("UAICode" not in headers and "UAI" not in headers):
print("\033[91mERROR: UAICode or depositAmount missing in CSV headers\033[0m")
return
data: dict[str, Decimal] = {}
# sometimes we get 1 row per institution and sometimes 1 row per class.
for row in csv_rows:
# try to get the UAI
uai_header = "UAI" if "UAI" in headers else "UAICode"
uai = row[uai_header].strip()
# try to get the amount
if "Crédits de dépenses" in headers or "depositAmount" in headers:
amount_header = "depositAmount" if "depositAmount" in headers else "Crédits de dépenses"
amount = Decimal(row[amount_header])
elif "montant par élève" in headers and "Effectif" in headers:
amount = Decimal(row["Effectif"]) * Decimal(row["montant par élève"])
else:
print("\033[91mERROR: Now way to get the amount found\033[0m")
return

if uai in data:
data[uai] += amount
else:
data[uai] = amount

institution_api.import_deposit_institution_data(
data=data,
educational_year=educational_year,
ministry=educational_models.Ministry[ministry],
conflict=conflict,
final=final,
commit=not dry_run,
)
institution_api.import_deposit_institution_csv(
path=path, year=year, ministry=ministry, conflict=conflict, final=final, commit=not dry_run
)


@blueprint.cli.command("synchronize_venues_from_adage_cultural_partners")
Expand Down
18 changes: 10 additions & 8 deletions api/tests/core/educational/api/test_import_deposit_institution_deposit_data.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,17 +80,19 @@ def test_institution_not_in_adage(self):
ansco = educational_factories.EducationalYearFactory()
data = {"pouet": 1250}

import_deposit_institution_data(
data=data,
educational_year=ansco,
ministry=educational_models.Ministry.EDUCATION_NATIONALE,
final=False,
conflict="crash",
commit=True,
)
with pytest.raises(ValueError) as exception:
import_deposit_institution_data(
data=data,
educational_year=ansco,
ministry=educational_models.Ministry.EDUCATION_NATIONALE,
final=False,
conflict="crash",
commit=True,
)

assert educational_models.EducationalInstitution.query.count() == 0
assert educational_models.EducationalDeposit.query.count() == 0
assert str(exception.value) == "UAIs not found in adage: ['pouet']"

def test_deposit_alread_in_ministry_replace(self) -> None:
ansco = educational_factories.EducationalYearFactory()
Expand Down
Loading