From e6ac4ce26cf9176cd09d817154d4afca6df0cfda Mon Sep 17 00:00:00 2001 From: Jairo Matos Da Rocha Date: Thu, 3 Oct 2024 18:22:31 -0300 Subject: [PATCH] limit 5MB --- app/api/task.py | 5 +- app/oauth2.py | 2 +- app/utils/file.py | 119 +++++++++++++++++++++++++++++++++------------- 3 files changed, 92 insertions(+), 34 deletions(-) diff --git a/app/api/task.py b/app/api/task.py index 6462f02..358d41e 100644 --- a/app/api/task.py +++ b/app/api/task.py @@ -41,6 +41,9 @@ async def savegeom_gdf( files: List[UploadFile] = File(...), user_data: UserInfo = Depends(has_role(['savegeom'])) # This should be a function that retrieves user data from the request. ): + for file in files: + if file.size > 5242880 : + raise HTTPException(status_code=413, detail="File size exceeds 5MB.") return __savegeom__(check_geofiles(files), {'name':name,'email':email}, user_data) @@ -79,6 +82,6 @@ def __savegeom__(gdf: gpd.GeoDataFrame, user: User,user_data: UserInfo): 'geojson':__checkgeom__(gdf), 'request_user': user_data } - logger.info(f"Starting task with payload: {dict_payload}") + #logger.info(f"Starting task with payload: {dict_payload}") task = gee_get_index_pasture.delay(dict_payload) return JSONResponse({"task_id": task.id}) \ No newline at end of file diff --git a/app/oauth2.py b/app/oauth2.py index f522789..5b3f3d9 100644 --- a/app/oauth2.py +++ b/app/oauth2.py @@ -55,7 +55,7 @@ async def get_payload(token: str = Security(oauth2_scheme)) -> dict: token ) except Exception as e: - logger.exception(f"Error decoding token: {e}") + #logger.exception(f"Error decoding token: {e}") raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, detail=str(e), # "Invalid authentication credentials", diff --git a/app/utils/file.py b/app/utils/file.py index c00244c..fb2c9b6 100644 --- a/app/utils/file.py +++ b/app/utils/file.py @@ -6,6 +6,45 @@ import geopandas as gpd from app.config import logger +MINIMAL_SHP_SUFFIX = [ + ".shp", # Geometria dos vetores + ".shx", # Índice de geometria + ".dbf", # Dados tabulares + ".prj", # Sistema de coordenadas e projeção + ] +FULL_SHP = [ + *MINIMAL_SHP_SUFFIX, + ".cpg", # Codificação de caracteres + ".sbn", # Índice espacial + ".sbx", # Arquivo auxiliar para índice espacial + ".xml", # Metadados em formato XML + ".qix", # Índice espacial (gerado por software) + ".aih", # Índice de atributos para .dbf + ".ain", # Arquivo auxiliar para índice de atributos + ".qmd" # Extensão adicional (se aplicável) + ] + +ZIP_READ = [ + '.kml', + '.kmz', + '.gpkg', + '.geojson', + '.topojson', + '.shp' +] + + +GEOFILES = [ + '.zip', + '.kml', + '.kmz', + '.gpkg', + '.geojson', + '.topojson', + *FULL_SHP +] + + def valid_file_geo(content_type): if content_type in [ @@ -26,22 +65,14 @@ def valid_file_geo(content_type): raise HTTPException(status_code=415, detail=f'Invalid file type: {content_type}') def valid_extension_shp(file): - if file.suffix in [ - ".shp", # Geometria dos vetores - ".shx", # Índice de geometria - ".dbf", # Dados tabulares - ".prj", # Sistema de coordenadas e projeção - ".cpg", # Codificação de caracteres - ".sbn", # Índice espacial - ".sbx", # Arquivo auxiliar para índice espacial - ".xml", # Metadados em formato XML - ".qix", # Índice espacial (gerado por software) - ".aih", # Índice de atributos para .dbf - ".ain", # Arquivo auxiliar para índice de atributos - ".qmd" # Extensão adicional (se aplicável) - ]: - return True - raise HTTPException(status_code=415, detail=f'Invalid file type: {file.suffix}') + if file.suffix.lower() in FULL_SHP: + return file.suffix.lower() + raise HTTPException(status_code=415, detail=f'Invalid file type: {file.suffix.lower()}') + +def valid_extension(file): + if file.suffix.lower() in GEOFILES: + return file.suffix.lower() + raise HTTPException(status_code=415, detail=f'Invalid file type: {file.suffix.lower()}') def check_geofiles(files): @@ -49,21 +80,27 @@ def check_geofiles(files): logger.info(f"Saving files to {tmpdirname}") logger.info(files) if len(files) == 1: + filename = Path(files[0].filename) valid_file_geo(files[0].content_type) + valid_extension(filename) + if filename.suffix.lower() in FULL_SHP: + minal_shape = set(MINIMAL_SHP_SUFFIX) - set([filename.suffix.lower()]) + raise HTTPException(status_code=400, detail=f'Missing files: {minal_shape}') + file = files[0] else: extensions = [] for f in files: valid_file_geo(f.content_type) valid_extension_shp(Path(f.filename)) - extensions.append(Path(f.filename).suffix) + extensions.append(Path(f.filename).suffix.lower()) with open(f'{tmpdirname}/{f.filename}', 'wb') as buffer: shutil.copyfileobj(f.file, buffer) - minal_shape =set(['.shp', '.shx', '.dbf', '.prj']) - set(extensions) + minal_shape = set(MINIMAL_SHP_SUFFIX) - set(extensions) if len(minal_shape) > 0: raise HTTPException(status_code=400, detail=f'Missing files: {minal_shape}') file = str(get_geofile(tmpdirname)) - return read_file(file) + return read_file(file, tmpdirname) def read_kml(file): @@ -75,16 +112,15 @@ def read_kml(file): return gpd.read_file(file, driver='KML') -def read_kmz(file): +def read_kmz(file,tmpdirname): import fiona - with tempfile.TemporaryDirectory() as tmpdirname: - kmz = ZipFile(file, 'r') - kmz.extract('doc.kml',tmpdirname) - try: - gdf = gpd.read_file(f'{tmpdirname}/doc.kml') - except Exception as e: - gpd.io.file.fiona.drvsupport.supported_drivers['KML'] = 'rw' - gdf = gpd.read_file(f'{tmpdirname}/doc.kml') + kmz = ZipFile(file, 'r') + kmz.extract('doc.kml',tmpdirname) + try: + gdf = gpd.read_file(f'{tmpdirname}/doc.kml') + except Exception as e: + gpd.io.file.fiona.drvsupport.supported_drivers['KML'] = 'rw' + gdf = gpd.read_file(f'{tmpdirname}/doc.kml') return gdf @@ -94,19 +130,38 @@ def read_gpd(file): def get_geofile(dirname): for file in Path(dirname).glob('*'): - if file.suffix in ['.shp']: + if file.suffix.lower() in ZIP_READ: return file -def read_file(file): +def read_file(file,tmpdirname): if isinstance(file, str): gdf = gpd.read_file(file) else: - match Path(file.filename).suffix.capitalize(): + match Path(file.filename).suffix.lower(): case '.kml': gdf = read_kml(file.file) case '.kmz': - gdf = read_kmz(file.file) + gdf = read_kmz(file.file, tmpdirname) + case '.zip': + dots = [] + with ZipFile(file.file) as zfile: + for f in zfile.namelist(): + logger.info(f'd {f}') + dots.append(valid_extension(Path(f))) + zfile.extractall(tmpdirname) + dots = [f for f in dots if f in ZIP_READ] + if len(dots) == 0 or len(dots) > 1: + raise HTTPException(status_code=400, detail='No valid files in the ZIP archive') + filename = get_geofile(tmpdirname) + + match Path(filename).suffix.lower(): + case '.kml': + gdf = read_kml(filename) + case '.kmz': + gdf = read_kmz(filename) + case _: + gdf = gpd.read_file(filename) case _: gdf = read_gpd(file.file)