From 9c0bc843bcfa55ccbebde800e2e2d288de3b96bb Mon Sep 17 00:00:00 2001 From: Jairo Matos Da Rocha Date: Mon, 30 Sep 2024 14:49:19 -0300 Subject: [PATCH] new work --- docker-compose.yml | 2 - worker.py | 193 +---------------------------- workers/email.py | 36 ++++++ workers/gee.py | 193 +++++++++++++++++++++++++++++ workers/models/email.py | 9 ++ workers/render.py | 11 ++ workers/utils/emial.py | 6 + {app => workers}/utils/gee2chat.py | 0 8 files changed, 259 insertions(+), 191 deletions(-) create mode 100644 workers/email.py create mode 100644 workers/gee.py create mode 100644 workers/models/email.py create mode 100644 workers/render.py create mode 100644 workers/utils/emial.py rename {app => workers}/utils/gee2chat.py (100%) diff --git a/docker-compose.yml b/docker-compose.yml index cf07cb6..eaf9b6d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,9 +13,7 @@ services: - SERVER_URL=${SERVER_URL} - CLIENT_ID=${CLIENT_ID} - CLIENT_SECRET=${CLIENT_SECRET} - - ADMIN_CLIENT_SECRET=${ADMIN_CLIENT_SECRET} - REALM=${REALM} - - CALLBACK_URI=${CALLBACK_URI} container_name: jobsgee privileged: true ports: diff --git a/worker.py b/worker.py index 6fddd59..086e3e4 100644 --- a/worker.py +++ b/worker.py @@ -1,14 +1,10 @@ import os -import json - from celery import Celery -from app.utils.gee2chat import get_chat_pasture, get_chat_pasture_vigor -from app.models.payload import PayloadSaveGeojson, ResultPayload +from workers.gee import task_index_pasture +from app.models.payload import ResultPayload from celery.utils.log import get_task_logger -from pymongo import MongoClient -import ee -import geemap + logger = get_task_logger(__name__) @@ -21,187 +17,6 @@ @celery.task(name="gee_get_index_pasture",bind=True) def gee_get_index_pasture(self, payload: ResultPayload): - geojson = payload.get('geojson') - def gee_credentials(private_key_file): - data = json.load(open(private_key_file)) - #logger.info(data) - gee_account = data['client_email'] - return ee.ServiceAccountCredentials(gee_account, private_key_file) - - service_account_file = '/var/sec/gee.json' - - - ee.Initialize(gee_credentials(service_account_file)) - - geometry = geemap.geojson_to_ee(geojson) - - exprecion = {'L8':{ - 'CAI': "(b('SR_B7') / b('SR_B6'))", - 'NDVI': "(b('SR_B5') - b('SR_B4')) / (b('SR_B5') + b('SR_B4'))", - 'NDWI': "(b('SR_B5') - b('SR_B6')) / (b('SR_B5') + b('SR_B6'))", - 'eq':2720, - 'bands': ["B3","B4","B5","B6","B7","NDVI","NDWI","CAI"] - }, - 'L5_7':{ - 'CAI': "(b('SR_B7') / b('SR_B5'))", - 'NDVI': "(b('SR_B4') - b('SR_B3')) / (b('SR_B4') + b('SR_B3'))", - 'NDWI': "(b('SR_B4') - b('SR_B5')) / (b('SR_B4') + b('SR_B5'))", - 'bands':["B2","B3","B4","B5","B7","NDVI","NDWI","CAI"], - 'eq':672 - }} - - pastue_years = [] - pastue_vigor_years = [] - for i in range(1985,2024): - pastue_years.append(ee.Image(f'users/lapig/pasture_col09/pasture_br_Y{i}_COL9_atlas').set('year',i)) - if i >= 2000: - pastue_vigor_years.append(ee.Image(f'users/lapig/pasture_vigor_col09/cvp_pasture_br_LAPIG_Y{i}_ATLAS').set('year',i)) - - pasture = ee.ImageCollection.fromImages(pastue_years) - pasture_vigor = ee.ImageCollection.fromImages(pastue_vigor_years) - - # Applies scaling factors. - def applyScaleFactors(image): - return image.select('SR_B.').multiply(0.0000275).add(-0.2); - - - ls05 = ee.ImageCollection("LANDSAT/LT05/C02/T1_L2") - ls07 = ee.ImageCollection('LANDSAT/LE07/C02/T1_L2') - ls08 = ee.ImageCollection('LANDSAT/LC08/C02/T1_L2') - - - def index_landsat57(image): - ndvi = image.expression(exprecion['L5_7']['NDVI']).rename('NDVI') - cai = image.expression(exprecion['L5_7']['CAI']).rename('CAI') - ndwi = image.expression(exprecion['L5_7']['NDWI']).rename('NWDI') - return image.addBands([ndvi,cai,ndwi]) - - def index_landsat8(image): - ndvi = image.expression(exprecion['L8']['NDVI']).rename('NDVI') - cai = image.expression(exprecion['L8']['CAI']).rename('CAI') - ndwi = image.expression(exprecion['L8']['NDWI']).rename('NWDI') - return image.addBands([ndvi,cai,ndwi]) - - def get_index(year): - return ee.Algorithms.If(year.gte(1985).And(year.lt(2012)), - (ls05.filterDate(ee.String(year).cat('-01-01'),ee.String(year).cat('-12-31')) - .filterBounds(geometry) - .map(applyScaleFactors).map(index_landsat57).mean().set('year',year).set('sat',5)), - ee.Algorithms.If(year.gte(2012).And(year.lt(2013)), - (ls07.filterDate(ee.String(year).cat('-01-01'),ee.String(year).cat('-12-31')) - .filterBounds(geometry) - .map(applyScaleFactors).map(index_landsat57).mean().set('year',year).set('sat',7)), - - (ls08.filterDate(ee.String(year).cat('-01-01'),ee.String(year).cat('-12-31')) - .filterBounds(geometry) - .map(applyScaleFactors).map(index_landsat8).mean().set('year',year).set('sat',8)) - ) - ) - - def get_precipitation(year): - dataset = (ee.ImageCollection('UCSB-CHG/CHIRPS/DAILY') - .filterBounds(geometry) - .filter(ee.Filter.date(ee.String(year).cat('-01-01'),ee.String(year).cat('-12-31'))) - ) - return dataset.select('precipitation').sum().set('year',year) - - def pasture_reduce(img): - year = img.get('year') - index = ee.Image(get_index(ee.Number(year))).updateMask(img) - precipitation = ee.Image(get_precipitation(ee.Number(year))).updateMask(img) - - - area_pixel = ee.Image.pixelArea().divide(10000).updateMask(img).rename('area_ha') - - result = area_pixel.reduceRegion(**{ - "reducer": ee.Reducer.sum(), - "geometry": geometry, - "scale": 30, - "maxPixels": 10e9 }) - - result_index = index.select(["NDVI","CAI","NWDI"]).addBands(precipitation).reduceRegion(**{ - "reducer": ee.Reducer.mean(), - "geometry": geometry, - "scale": 30, - "maxPixels": 10e9 }) - - - return ee.Feature(None,ee.Dictionary({ - "area_ha":result.get('area_ha'), - "indexs":result_index, - "year":year})) - - def pasute_vigor_reduce(img): - year = img.get('year') - index = ee.Image(get_index(ee.Number(year))).updateMask(img) - - area_pixel = ee.Image.pixelArea().divide(10000).rename('area_ha').addBands(img.select('b1')) - result = area_pixel.reduceRegion(**{ - "reducer": ee.Reducer.sum().group(**{ - "groupField": 1, - "groupName": 'class', - }), - "geometry": geometry, - "scale": 30, - "maxPixels": 10e9 }) - - result_ndvi = index.select('NDVI').addBands(img.select('b1')).reduceRegion(**{ - "reducer": ee.Reducer.mean().group(**{ - "groupField": 1, - "groupName": 'class', - }), - "geometry": geometry, - "scale": 30, - "maxPixels": 10e9 }) - - result_cai = index.select('CAI').addBands(img.select('b1')).reduceRegion(**{ - "reducer": ee.Reducer.mean().group(**{ - "groupField": 1, - "groupName": 'class', - }), - "geometry": geometry, - "scale": 30, - "maxPixels": 10e9 }) - - result_ndwi = index.select('NWDI').addBands(img.select('b1')).reduceRegion(**{ - "reducer": ee.Reducer.mean().group(**{ - "groupField": 1, - "groupName": 'class', - }), - "geometry": geometry, - "scale": 30, - "maxPixels": 10e9 }) - - - - return ee.Feature(None,ee.Dictionary({ - "area_ha":result, - "ndiv":result_ndvi, - "cai":result_cai, - "ndwi":result_ndwi, - "year":year - - })) - - - result_pasture = pasture.map(pasture_reduce) - - result_pasture_vigor = pasture_vigor.map(pasute_vigor_reduce) - - result = { - '_id':self.request.id, - **payload, - 'pasture':get_chat_pasture(result_pasture.getInfo()['features']), - 'pasture_vigor':get_chat_pasture_vigor(result_pasture_vigor.getInfo()['features']), - } - - with MongoClient(os.environ.get("MONGOURI", "mongodb://mongodbjobs:27017")) as cliente: - # Seleciona o banco de dados e a coleção - banco_de_dados = cliente['lapig-task'] - colecao = banco_de_dados['results'] - - # Insere o documento na coleção - resultado = colecao.insert_one(result) - return result + return task_index_pasture(self.request.id, payload) \ No newline at end of file diff --git a/workers/email.py b/workers/email.py new file mode 100644 index 0000000..58241be --- /dev/null +++ b/workers/email.py @@ -0,0 +1,36 @@ +import os +import smtplib, ssl +from email.mime.text import MIMEText +from email.mime.multipart import MIMEMultipart + + +from workers.models.email import SendEmail +from workers.render import template +from workers.utils.emial import html_to_text + + +def send_email(payload: SendEmail): + + sender_email = os.environ.get('EMAIL_SENDER') + password = os.environ.get('EMAIL_PASSWORD') + receiver_email = payload['receiver_email'] + + + message = MIMEMultipart("alternative") + message["Subject"] = "multipart test" + message["From"] = sender_email + message["To"] = receiver_email + + to_render = payload['message'] + html = template.get_template(to_render['templet_name']).render(to_render['content']) + + message.attach(MIMEText(html_to_text(html), "plain")) + message.attach(MIMEText(html, "html")) + + # Create secure connection with server and send email + context = ssl.create_default_context() + with smtplib.SMTP_SSL(os.environ.get('SERVER_EMAIL'), 465, context=context) as server: + server.login(sender_email, password) + server.sendmail( + sender_email, receiver_email, message.as_string() + ) \ No newline at end of file diff --git a/workers/gee.py b/workers/gee.py new file mode 100644 index 0000000..4362558 --- /dev/null +++ b/workers/gee.py @@ -0,0 +1,193 @@ +from workers.utils.gee2chat import get_chat_pasture, get_chat_pasture_vigor +from app.models.payload import ResultPayload +from celery.utils.log import get_task_logger +from pymongo import MongoClient +import ee +import geemap +import json + +logger = get_task_logger(__name__) + +def task_index_pasture(task_id: str, payload: ResultPayload): + geojson = payload.get('geojson') + def gee_credentials(private_key_file): + data = json.load(open(private_key_file)) + #logger.info(data) + gee_account = data['client_email'] + return ee.ServiceAccountCredentials(gee_account, private_key_file) + + service_account_file = '/var/sec/gee.json' + + + ee.Initialize(gee_credentials(service_account_file)) + + geometry = geemap.geojson_to_ee(geojson) + + exprecion = {'L8':{ + 'CAI': "(b('SR_B7') / b('SR_B6'))", + 'NDVI': "(b('SR_B5') - b('SR_B4')) / (b('SR_B5') + b('SR_B4'))", + 'NDWI': "(b('SR_B5') - b('SR_B6')) / (b('SR_B5') + b('SR_B6'))", + 'eq':2720, + 'bands': ["B3","B4","B5","B6","B7","NDVI","NDWI","CAI"] + }, + 'L5_7':{ + 'CAI': "(b('SR_B7') / b('SR_B5'))", + 'NDVI': "(b('SR_B4') - b('SR_B3')) / (b('SR_B4') + b('SR_B3'))", + 'NDWI': "(b('SR_B4') - b('SR_B5')) / (b('SR_B4') + b('SR_B5'))", + 'bands':["B2","B3","B4","B5","B7","NDVI","NDWI","CAI"], + 'eq':672 + }} + + pastue_years = [] + pastue_vigor_years = [] + for i in range(1985,2024): + pastue_years.append(ee.Image(f'users/lapig/pasture_col09/pasture_br_Y{i}_COL9_atlas').set('year',i)) + if i >= 2000: + pastue_vigor_years.append(ee.Image(f'users/lapig/pasture_vigor_col09/cvp_pasture_br_LAPIG_Y{i}_ATLAS').set('year',i)) + + pasture = ee.ImageCollection.fromImages(pastue_years) + pasture_vigor = ee.ImageCollection.fromImages(pastue_vigor_years) + + # Applies scaling factors. + def applyScaleFactors(image): + return image.select('SR_B.').multiply(0.0000275).add(-0.2); + + + ls05 = ee.ImageCollection("LANDSAT/LT05/C02/T1_L2") + ls07 = ee.ImageCollection('LANDSAT/LE07/C02/T1_L2') + ls08 = ee.ImageCollection('LANDSAT/LC08/C02/T1_L2') + + + def index_landsat57(image): + ndvi = image.expression(exprecion['L5_7']['NDVI']).rename('NDVI') + cai = image.expression(exprecion['L5_7']['CAI']).rename('CAI') + ndwi = image.expression(exprecion['L5_7']['NDWI']).rename('NWDI') + return image.addBands([ndvi,cai,ndwi]) + + def index_landsat8(image): + ndvi = image.expression(exprecion['L8']['NDVI']).rename('NDVI') + cai = image.expression(exprecion['L8']['CAI']).rename('CAI') + ndwi = image.expression(exprecion['L8']['NDWI']).rename('NWDI') + return image.addBands([ndvi,cai,ndwi]) + + def get_index(year): + return ee.Algorithms.If(year.gte(1985).And(year.lt(2012)), + (ls05.filterDate(ee.String(year).cat('-01-01'),ee.String(year).cat('-12-31')) + .filterBounds(geometry) + .map(applyScaleFactors).map(index_landsat57).mean().set('year',year).set('sat',5)), + ee.Algorithms.If(year.gte(2012).And(year.lt(2013)), + (ls07.filterDate(ee.String(year).cat('-01-01'),ee.String(year).cat('-12-31')) + .filterBounds(geometry) + .map(applyScaleFactors).map(index_landsat57).mean().set('year',year).set('sat',7)), + + (ls08.filterDate(ee.String(year).cat('-01-01'),ee.String(year).cat('-12-31')) + .filterBounds(geometry) + .map(applyScaleFactors).map(index_landsat8).mean().set('year',year).set('sat',8)) + ) + ) + + def get_precipitation(year): + dataset = (ee.ImageCollection('UCSB-CHG/CHIRPS/DAILY') + .filterBounds(geometry) + .filter(ee.Filter.date(ee.String(year).cat('-01-01'),ee.String(year).cat('-12-31'))) + ) + return dataset.select('precipitation').sum().set('year',year) + + def pasture_reduce(img): + year = img.get('year') + index = ee.Image(get_index(ee.Number(year))).updateMask(img) + precipitation = ee.Image(get_precipitation(ee.Number(year))).updateMask(img) + + + area_pixel = ee.Image.pixelArea().divide(10000).updateMask(img).rename('area_ha') + + result = area_pixel.reduceRegion(**{ + "reducer": ee.Reducer.sum(), + "geometry": geometry, + "scale": 30, + "maxPixels": 10e9 }) + + result_index = index.select(["NDVI","CAI","NWDI"]).addBands(precipitation).reduceRegion(**{ + "reducer": ee.Reducer.mean(), + "geometry": geometry, + "scale": 30, + "maxPixels": 10e9 }) + + + return ee.Feature(None,ee.Dictionary({ + "area_ha":result.get('area_ha'), + "indexs":result_index, + "year":year})) + + def pasute_vigor_reduce(img): + year = img.get('year') + index = ee.Image(get_index(ee.Number(year))).updateMask(img) + + area_pixel = ee.Image.pixelArea().divide(10000).rename('area_ha').addBands(img.select('b1')) + result = area_pixel.reduceRegion(**{ + "reducer": ee.Reducer.sum().group(**{ + "groupField": 1, + "groupName": 'class', + }), + "geometry": geometry, + "scale": 30, + "maxPixels": 10e9 }) + + result_ndvi = index.select('NDVI').addBands(img.select('b1')).reduceRegion(**{ + "reducer": ee.Reducer.mean().group(**{ + "groupField": 1, + "groupName": 'class', + }), + "geometry": geometry, + "scale": 30, + "maxPixels": 10e9 }) + + result_cai = index.select('CAI').addBands(img.select('b1')).reduceRegion(**{ + "reducer": ee.Reducer.mean().group(**{ + "groupField": 1, + "groupName": 'class', + }), + "geometry": geometry, + "scale": 30, + "maxPixels": 10e9 }) + + result_ndwi = index.select('NWDI').addBands(img.select('b1')).reduceRegion(**{ + "reducer": ee.Reducer.mean().group(**{ + "groupField": 1, + "groupName": 'class', + }), + "geometry": geometry, + "scale": 30, + "maxPixels": 10e9 }) + + + + return ee.Feature(None,ee.Dictionary({ + "area_ha":result, + "ndiv":result_ndvi, + "cai":result_cai, + "ndwi":result_ndwi, + "year":year + + })) + + + result_pasture = pasture.map(pasture_reduce) + + result_pasture_vigor = pasture_vigor.map(pasute_vigor_reduce) + + result = { + '_id':task_id, + **payload, + 'pasture':get_chat_pasture(result_pasture.getInfo()['features']), + 'pasture_vigor':get_chat_pasture_vigor(result_pasture_vigor.getInfo()['features']), + } + + with MongoClient(os.environ.get("MONGOURI", "mongodb://mongodbjobs:27017")) as cliente: + # Seleciona o banco de dados e a coleção + banco_de_dados = cliente['lapig-task'] + colecao = banco_de_dados['results'] + + # Insere o documento na coleção + resultado = colecao.insert_one(result) + return result \ No newline at end of file diff --git a/workers/models/email.py b/workers/models/email.py new file mode 100644 index 0000000..232b3e4 --- /dev/null +++ b/workers/models/email.py @@ -0,0 +1,9 @@ +from pydantic import BaseModel, EmailStr +class MessageHTML(BaseModel): + template_name: str + content: dict + +class SendEmail(BaseModel): + receiver_email: EmailStr + subject: str + message: MessageHTML diff --git a/workers/render.py b/workers/render.py new file mode 100644 index 0000000..cf4e4ef --- /dev/null +++ b/workers/render.py @@ -0,0 +1,11 @@ +from pathlib import Path +from jinja2 import Environment, FileSystemLoader + +# Diretório do script atual +current_directory = Path(__file__).resolve().parent + +# Diretório dos templates +templates_directory = current_directory / 'templates' + +# Configurando o Jinja2 para carregar templates do diretório +template = Environment(loader=FileSystemLoader(str(templates_directory))) \ No newline at end of file diff --git a/workers/utils/emial.py b/workers/utils/emial.py new file mode 100644 index 0000000..1a4f984 --- /dev/null +++ b/workers/utils/emial.py @@ -0,0 +1,6 @@ +from bs4 import BeautifulSoup + + +def html_to_text(html): + soup = BeautifulSoup(html) + return soup.get_text() diff --git a/app/utils/gee2chat.py b/workers/utils/gee2chat.py similarity index 100% rename from app/utils/gee2chat.py rename to workers/utils/gee2chat.py