From 79735459e5a8f7b830868d2846b18ed77c54230d Mon Sep 17 00:00:00 2001 From: Jairo Matos Da Rocha Date: Thu, 3 Oct 2024 14:12:18 -0300 Subject: [PATCH] file upload --- app/api/task.py | 83 +++++++++++++++++++++------------- app/oauth2.py | 1 - app/utils/file.py | 113 ++++++++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 1 + uv.lock | 46 +++++++++++++++++++ 5 files changed, 212 insertions(+), 32 deletions(-) create mode 100644 app/utils/file.py diff --git a/app/api/task.py b/app/api/task.py index cf1b71c..6462f02 100644 --- a/app/api/task.py +++ b/app/api/task.py @@ -1,56 +1,51 @@ +from pathlib import Path +import shutil +import tempfile +from typing import List from fastapi.responses import JSONResponse import geopandas as gpd +from pydantic import EmailStr from app.models.oauth2 import UserInfo from app.oauth2 import has_role +from app.utils.file import check_geofiles from worker import gee_get_index_pasture -from app.models.payload import PayloadSaveGeojson +from app.models.payload import PayloadSaveGeojson, User import os -from fastapi import APIRouter, Depends, HTTPException, Request, Query +from fastapi import APIRouter, Depends, File, HTTPException, Request, Query, UploadFile from app.config import logger from celery.result import AsyncResult router = APIRouter() -@router.post("/savegeom" ) -async def savegeom( +@router.post("/savegeom/geojson" ) +async def savegeom_geojson( payload: PayloadSaveGeojson, - request: Request, crs: int = Query(4326, description="EPSG code for the geometry"), user_data: UserInfo = Depends(has_role(['savegeom'])) # This should be a function that retrieves user data from the request. ): - - MAXHECTARES = os.environ.get('MAXHECTARES',40_000) geojson = payload.dict().get('geojson',gpd.GeoDataFrame()) + user = payload.dict().get('user') logger.info(f"Received payload: {geojson}") - try: - gdf = gpd.GeoDataFrame.from_features(geojson, crs=crs) - if gdf.empty or len(gdf) > 1: - return HTTPException(status_code=400, detail="Empty GeoDataFrame or more than one feature.") - if gdf.geometry.type[0] != "Polygon": - return HTTPException(status_code=400, detail="Geometry must be a Polygon.") - if gdf.to_crs(5880).area.iloc[0] / 10_000 > MAXHECTARES: - return HTTPException(status_code=400, detail=f"Geometry area must be less than {MAXHECTARES} hectares.") - logger.info('Geometry is valid') - logger.info(user_data) - try: - dict_payload = { - **payload.dict(), - 'request_user': user_data - } - task = gee_get_index_pasture.delay(dict_payload) - except Exception as e: - logger.exception(f"Failed to create task: {e}") - raise HTTPException(status_code=400, detail="Failed to create task.") - except Exception as e: - logger.error(f"{e}") - raise HTTPException(status_code=400, detail=e) + gdf = gpd.GeoDataFrame.from_features(geojson, crs=crs) + return __savegeom__(gdf, user, user_data) + +@router.post("/savegeom/file" ) +async def savegeom_gdf( + name: str, + email:EmailStr, + files: List[UploadFile] = File(...), + user_data: UserInfo = Depends(has_role(['savegeom'])) # This should be a function that retrieves user data from the request. +): + return __savegeom__(check_geofiles(files), {'name':name,'email':email}, user_data) + + + - return JSONResponse({"task_id": task.id}) @router.get("/status/{task_id}") def get_status(task_id): @@ -60,4 +55,30 @@ def get_status(task_id): "task_status": task_result.status, "task_result": task_result.result } - return JSONResponse(result) \ No newline at end of file + return JSONResponse(result) + + +def __checkgeom__(gdf: gpd.GeoDataFrame): + MAXHECTARES = os.environ.get('MAXHECTARES',40_000) + if gdf.empty: + raise HTTPException(status_code=400, detail="Empty GeoDataFrame.") + if len(gdf) > 1: + raise HTTPException(status_code=400, detail="GeoDataFrame must have only one geometry.") + if gdf.geometry.type[0] != "Polygon": + raise HTTPException(status_code=400, detail="Geometry must be a Polygon.") + if gdf.to_crs(5880).area.iloc[0] / 10_000 > MAXHECTARES: + raise HTTPException(status_code=400, detail=f"Geometry area must be less than {MAXHECTARES} hectares.") + logger.info('Geometry is valid') + return gdf.to_crs(4326).to_geo_dict() + + + +def __savegeom__(gdf: gpd.GeoDataFrame, user: User,user_data: UserInfo): + dict_payload = { + 'user':user, + 'geojson':__checkgeom__(gdf), + 'request_user': user_data + } + logger.info(f"Starting task with payload: {dict_payload}") + task = gee_get_index_pasture.delay(dict_payload) + return JSONResponse({"task_id": task.id}) \ No newline at end of file diff --git a/app/oauth2.py b/app/oauth2.py index 8ae368c..f522789 100644 --- a/app/oauth2.py +++ b/app/oauth2.py @@ -49,7 +49,6 @@ async def get_idp_public_key(): # Get the payload/token from keycloak async def get_payload(token: str = Security(oauth2_scheme)) -> dict: key= await get_idp_public_key() - logger.info(f"token {token}") try: return keycloak_openid.decode_token( diff --git a/app/utils/file.py b/app/utils/file.py new file mode 100644 index 0000000..c00244c --- /dev/null +++ b/app/utils/file.py @@ -0,0 +1,113 @@ +import shutil +import tempfile +from zipfile import ZipFile +from pathlib import Path +from fastapi import HTTPException +import geopandas as gpd +from app.config import logger + +def valid_file_geo(content_type): + + if content_type in [ + 'application/text', + 'application/xml', + 'text/plain', + 'application/geo+json', + 'application/geopackage+sqlite3', + 'application/octet-stream', + 'application/vnd.google-earth.kml+xml', + 'application/vnd.google-earth.kmz', + 'application/x-dbf', + 'application/x-esri-crs', + 'application/x-esri-shape', + 'application/zip' + ]: + return True + raise HTTPException(status_code=415, detail=f'Invalid file type: {content_type}') + +def valid_extension_shp(file): + if file.suffix in [ + ".shp", # Geometria dos vetores + ".shx", # Índice de geometria + ".dbf", # Dados tabulares + ".prj", # Sistema de coordenadas e projeção + ".cpg", # Codificação de caracteres + ".sbn", # Índice espacial + ".sbx", # Arquivo auxiliar para índice espacial + ".xml", # Metadados em formato XML + ".qix", # Índice espacial (gerado por software) + ".aih", # Índice de atributos para .dbf + ".ain", # Arquivo auxiliar para índice de atributos + ".qmd" # Extensão adicional (se aplicável) + ]: + return True + raise HTTPException(status_code=415, detail=f'Invalid file type: {file.suffix}') + + +def check_geofiles(files): + with tempfile.TemporaryDirectory() as tmpdirname: + logger.info(f"Saving files to {tmpdirname}") + logger.info(files) + if len(files) == 1: + valid_file_geo(files[0].content_type) + file = files[0] + else: + extensions = [] + for f in files: + valid_file_geo(f.content_type) + valid_extension_shp(Path(f.filename)) + extensions.append(Path(f.filename).suffix) + with open(f'{tmpdirname}/{f.filename}', 'wb') as buffer: + shutil.copyfileobj(f.file, buffer) + minal_shape =set(['.shp', '.shx', '.dbf', '.prj']) - set(extensions) + if len(minal_shape) > 0: + raise HTTPException(status_code=400, detail=f'Missing files: {minal_shape}') + file = str(get_geofile(tmpdirname)) + return read_file(file) + + +def read_kml(file): + import fiona + try: + return gpd.read_file(file, driver='KML') + except Exception as e: + gpd.io.file.fiona.drvsupport.supported_drivers['KML'] = 'rw' + return gpd.read_file(file, driver='KML') + + +def read_kmz(file): + import fiona + with tempfile.TemporaryDirectory() as tmpdirname: + kmz = ZipFile(file, 'r') + kmz.extract('doc.kml',tmpdirname) + try: + gdf = gpd.read_file(f'{tmpdirname}/doc.kml') + except Exception as e: + gpd.io.file.fiona.drvsupport.supported_drivers['KML'] = 'rw' + gdf = gpd.read_file(f'{tmpdirname}/doc.kml') + + return gdf + +def read_gpd(file): + return gpd.read_file(file) + + +def get_geofile(dirname): + for file in Path(dirname).glob('*'): + if file.suffix in ['.shp']: + return file + + +def read_file(file): + if isinstance(file, str): + gdf = gpd.read_file(file) + else: + match Path(file.filename).suffix.capitalize(): + case '.kml': + gdf = read_kml(file.file) + case '.kmz': + gdf = read_kmz(file.file) + case _: + gdf = read_gpd(file.file) + + return gdf \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index f381afa..b2f54ec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ dependencies = [ "earthengine-api>=1.1.2", "fastapi-keycloak-middleware>=1.1.0", "fastapi>=0.115.0", + "fiona>=1.10.1", "flower>=2.0.1", "geemap>=0.34.5", "geopandas>=1.0.1", diff --git a/uv.lock b/uv.lock index f2f003d..a1d3817 100644 --- a/uv.lock +++ b/uv.lock @@ -60,6 +60,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c7/80/9f608d13b4b3afcebd1dd13baf9551c95fc424d6390e4b1cfd7b1810cd06/async_property-0.2.2-py2.py3-none-any.whl", hash = "sha256:8924d792b5843994537f8ed411165700b27b2bd966cefc4daeefc1253442a9d7", size = 9546 }, ] +[[package]] +name = "attrs" +version = "24.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fc/0f/aafca9af9315aee06a89ffde799a10a582fe8de76c563ee80bbcdc08b3fb/attrs-24.2.0.tar.gz", hash = "sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346", size = 792678 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6a/21/5b6702a7f963e95456c0de2d495f67bf5fd62840ac655dc451586d23d39a/attrs-24.2.0-py3-none-any.whl", hash = "sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2", size = 63001 }, +] + [[package]] name = "beautifulsoup4" version = "4.12.3" @@ -253,6 +262,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/52/40/9d857001228658f0d59e97ebd4c346fe73e138c6de1bce61dc568a57c7f8/click_repl-0.3.0-py3-none-any.whl", hash = "sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812", size = 10289 }, ] +[[package]] +name = "cligj" +version = "0.7.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ea/0d/837dbd5d8430fd0f01ed72c4cfb2f548180f4c68c635df84ce87956cff32/cligj-0.7.2.tar.gz", hash = "sha256:a4bc13d623356b373c2c27c53dbd9c68cae5d526270bfa71f6c6fa69669c6b27", size = 9803 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/73/86/43fa9f15c5b9fb6e82620428827cd3c284aa933431405d1bcf5231ae3d3e/cligj-0.7.2-py3-none-any.whl", hash = "sha256:c1ca117dbce1fe20a5809dc96f01e1c2840f6dcc939b3ddbb1111bf330ba82df", size = 7069 }, +] + [[package]] name = "colorama" version = "0.4.6" @@ -477,6 +498,29 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ce/ad/500df28e424188c2d992fa9b754366f6564782712c72e9578df76dd7ee9b/fastapi_keycloak_middleware-1.1.0-py3-none-any.whl", hash = "sha256:cc5cba9b24dd297f8bb2a77207df66c903b05c411863cd4c6fc79152b9c7cde2", size = 22300 }, ] +[[package]] +name = "fiona" +version = "1.10.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "certifi" }, + { name = "click" }, + { name = "click-plugins" }, + { name = "cligj" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/51/e0/71b63839cc609e1d62cea2fc9774aa605ece7ea78af823ff7a8f1c560e72/fiona-1.10.1.tar.gz", hash = "sha256:b00ae357669460c6491caba29c2022ff0acfcbde86a95361ea8ff5cd14a86b68", size = 444606 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/73/ab/036c418d531afb74abe4ca9a8be487b863901fe7b42ddba1ba2fb0681d77/fiona-1.10.1-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:7338b8c68beb7934bde4ec9f49eb5044e5e484b92d940bc3ec27defdb2b06c67", size = 16114589 }, + { url = "https://files.pythonhosted.org/packages/ba/45/693c1cca53023aaf6e3adc11422080f5fa427484e7b85e48f19c40d6357f/fiona-1.10.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8c77fcfd3cdb0d3c97237965f8c60d1696a64923deeeb2d0b9810286cbe25911", size = 14754603 }, + { url = "https://files.pythonhosted.org/packages/dc/78/be204fb409b59876ef4658710a022794f16f779a3e9e7df654acc38b2104/fiona-1.10.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:537872cbc9bda7fcdf73851c91bc5338fca2b502c4c17049ccecaa13cde1f18f", size = 17223639 }, + { url = "https://files.pythonhosted.org/packages/7e/0d/914fd3c4c32043c2c512fa5021e83b2348e1b7a79365d75a0a37cb545362/fiona-1.10.1-cp312-cp312-win_amd64.whl", hash = "sha256:41cde2c52c614457e9094ea44b0d30483540789e62fe0fa758c2a2963e980817", size = 24464921 }, + { url = "https://files.pythonhosted.org/packages/c5/e0/665ce969cab6339c19527318534236e5e4184ee03b38cd474497ebd22f4d/fiona-1.10.1-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:a00b05935c9900678b2ca660026b39efc4e4b916983915d595964eb381763ae7", size = 16106571 }, + { url = "https://files.pythonhosted.org/packages/23/c8/150094fbc4220d22217f480cc67b6ee4c2f4324b4b58cd25527cd5905937/fiona-1.10.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f78b781d5bcbbeeddf1d52712f33458775dbb9fd1b2a39882c83618348dd730f", size = 14738178 }, + { url = "https://files.pythonhosted.org/packages/20/83/63da54032c0c03d4921b854111e33d3a1dadec5d2b7e741fba6c8c6486a6/fiona-1.10.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:29ceeb38e3cd30d91d68858d0817a1bb0c4f96340d334db4b16a99edb0902d35", size = 17221414 }, + { url = "https://files.pythonhosted.org/packages/60/14/5ef47002ef19bd5cfbc7a74b21c30ef83f22beb80609314ce0328989ceda/fiona-1.10.1-cp313-cp313-win_amd64.whl", hash = "sha256:15751c90e29cee1e01fcfedf42ab85987e32f0b593cf98d88ed52199ef5ca623", size = 24461486 }, +] + [[package]] name = "flower" version = "2.0.1" @@ -1027,6 +1071,7 @@ dependencies = [ { name = "earthengine-api" }, { name = "fastapi" }, { name = "fastapi-keycloak-middleware" }, + { name = "fiona" }, { name = "flower" }, { name = "geemap" }, { name = "geopandas" }, @@ -1053,6 +1098,7 @@ requires-dist = [ { name = "earthengine-api", specifier = ">=1.1.2" }, { name = "fastapi", specifier = ">=0.115.0" }, { name = "fastapi-keycloak-middleware", specifier = ">=1.1.0" }, + { name = "fiona", specifier = ">=1.10.1" }, { name = "flower", specifier = ">=2.0.1" }, { name = "geemap", specifier = ">=0.34.5" }, { name = "geopandas", specifier = ">=1.0.1" },