From 597d1e76b8a24aab5f1dc65ba14d2e6169f0b6e7 Mon Sep 17 00:00:00 2001 From: Juan Vinicius Date: Tue, 11 Aug 2020 14:09:01 -0300 Subject: [PATCH 1/4] Fix vectorial process bug(verification with id) --- .../vectorial_processing_intersection.sql | 31 +++++--- src/terrama2/impl/DataStoragerTable.cpp | 78 +++++++++++++++++++ src/terrama2/impl/DataStoragerTable.hpp | 1 + ...00806000000-create-table-ids-controller.js | 17 ++++ 4 files changed, 116 insertions(+), 11 deletions(-) create mode 100644 webapp/migrations/20200806000000-create-table-ids-controller.js diff --git a/share/terrama2/scripts/sql-functions/vectorial_processing_intersection.sql b/share/terrama2/scripts/sql-functions/vectorial_processing_intersection.sql index eb884fbb1..a0f1366e6 100644 --- a/share/terrama2/scripts/sql-functions/vectorial_processing_intersection.sql +++ b/share/terrama2/scripts/sql-functions/vectorial_processing_intersection.sql @@ -174,6 +174,10 @@ DECLARE number_of_attributes INTEGER; number_of_affected_rows BIGINT; + dynamic_last_id_inserted INTEGER; + finaltable_last_id_inserted INTEGER; + is_finaltable_exists_idcontroller BOOLEAN; + query TEXT; result RECORD; BEGIN @@ -197,13 +201,20 @@ BEGIN -- Getting date_column_from_dynamic_table EXECUTE format('SELECT get_date_column_from_dynamic_table(''%s'')', dynamic_table_name_handler) INTO date_column_from_dynamic_table; - -- Getting last time collected from analysis_metadata - EXECUTE format('SELECT COUNT(*) FROM terrama2.analysis_metadata WHERE key = ''last_analysis'' AND terrama2.analysis_metadata.analysis_id = %s', analysis_id) INTO number_of_rows_metadata; - - EXECUTE format('SELECT MAX(value::TIMESTAMP) FROM terrama2.analysis_metadata WHERE key = ''last_analysis'' AND terrama2.analysis_metadata.analysis_id = %s', analysis_id) INTO last_analysis; + -- Getting last id inserted in dynamic table + EXECUTE format('SELECT last_id FROM terrama2.table_ids_controller WHERE table_name = ''%s''', dynamic_table_name_handler) INTO dynamic_last_id_inserted; final_table := format('%s_%s', output_table_name, analysis_id); + EXECUTE format('SELECT EXISTS (SELECT * FROM terrama2.table_ids_controller where table_name = ''%s'')', final_table) INTO is_finaltable_exists_idcontroller; + + IF (is_finaltable_exists_idcontroller) THEN + EXECUTE format('SELECT last_id FROM terrama2.table_ids_controller WHERE table_name = ''%s''', final_table) INTO finaltable_last_id_inserted; + ELSE + EXECUTE format('INSERT INTO terrama2.table_ids_controller (table_name, last_id) VALUES(''%s'', 0)', final_table); + finaltable_last_id_inserted := 0; + END IF; + EXECUTE 'SELECT get_primary_key($1)' INTO pk_static_table USING static_table_name; EXECUTE 'SELECT get_primary_key($1)' INTO pk_dynamic_table USING dynamic_table_name_handler; @@ -219,12 +230,7 @@ BEGIN -- 6 this is the default number of attributes: mo nitored_id, intersect_id, execution_date, intersection_geom, calculated_area_ha, final_table_id; IF (is_table_exists AND number_of_columns = number_of_attributes) THEN - -- Creating analysis filter - IF number_of_rows_metadata > 0 THEN - analysis_filter := format(' %s.%s > ''%s'' ', dynamic_table_name_handler, date_column_from_dynamic_table, last_analysis); - ELSE - analysis_filter := '1 = 1'; - END IF; + analysis_filter := format(' %s.pid_%s > %s ', dynamic_table_name_handler, dynamic_table_name_handler, finaltable_last_id_inserted); date_filter := '1 = 1'; @@ -278,7 +284,7 @@ BEGIN GET DIAGNOSTICS number_of_affected_rows = ROW_COUNT; ELSE - analysis_filter := '1 = 1'; + analysis_filter := format(' %s.pid_%s > %s ', dynamic_table_name_handler, dynamic_table_name_handler, finaltable_last_id_inserted); date_filter := '1 = 1'; @@ -347,6 +353,9 @@ BEGIN EXECUTE query; END IF; + -- Refresh id number in ids controller + EXECUTE format('UPDATE terrama2.table_ids_controller SET last_id = %s WHERE table_name = ''%s''', dynamic_last_id_inserted, final_table); + query := format('ALTER TABLE %s ALTER COLUMN intersection_geom TYPE geometry(GEOMETRY, %s) USING ST_Transform(ST_SetSRID(intersection_geom,%s), %s)',final_table, dynamic_table_srid, dynamic_table_srid, dynamic_table_srid); EXECUTE query; diff --git a/src/terrama2/impl/DataStoragerTable.cpp b/src/terrama2/impl/DataStoragerTable.cpp index c417fa513..2aa69cdb4 100755 --- a/src/terrama2/impl/DataStoragerTable.cpp +++ b/src/terrama2/impl/DataStoragerTable.cpp @@ -327,6 +327,84 @@ void terrama2::core::DataStoragerTable::store(DataSetSeries series, DataSetPtr o transactorDestination->add(newDataSetType->getName(), series.syncDataSet->dataset().get(), {}); scopedTransaction.commit(); + + if(this->dataSeries_.get()->semantics.temporality == terrama2::core::DataSeriesTemporality::DYNAMIC || this->dataSeries_.get()->semantics.dataSeriesType == terrama2::core::DataSeriesType::VECTOR_PROCESSING_OBJECT) + { + storeDynamicQuantity(transactorDestination, destinationDataSetName); + } +} + +void terrama2::core::DataStoragerTable::storeDynamicQuantity(std::shared_ptr transactorDestination, std::string destinationDataSetName) const +{ + + double quantityTableController; + + boost::format queryQuantityTableController("SELECT last_id from terrama2.table_ids_controller where table_name = '" + destinationDataSetName +"'"); + + std::shared_ptr tempDataSetQuantTableContr(transactorDestination->query(queryQuantityTableController.str())); + + if(!tempDataSetQuantTableContr) + { + QString errMsg = QObject::tr("Can not find log message table name!"); + TERRAMA2_LOG_ERROR() << errMsg; + throw terrama2::core::LogException() << ErrorDescription(errMsg); + } + + if(!tempDataSetQuantTableContr->moveFirst()) + { + //CASE TABLE IS EMPTY + quantityTableController = 0; + boost::format queryInsertInitial("INSERT INTO terrama2.table_ids_controller (table_name, last_id) VALUES('%1%', %2%)"); + queryInsertInitial.bind_arg(1, destinationDataSetName); + queryInsertInitial.bind_arg(2, 0); + + transactorDestination->execute(queryInsertInitial.str()); + + transactorDestination->commit(); + } + else + { + tempDataSetQuantTableContr->moveFirst(); + quantityTableController = tempDataSetQuantTableContr->getInt32("last_id"); + } + + double quantityRealtable; + + //RETRIEVE ID FROM LAST DATA INSERTED + boost::format queryRealTable("SELECT pid_"+destinationDataSetName+" FROM (SELECT * FROM public." + destinationDataSetName + " ORDER BY pid_" + destinationDataSetName + " DESC LIMIT 1) AS last_data"); + + std::shared_ptr tempDataSetRealtable(transactorDestination->query(queryRealTable.str())); + + if(!tempDataSetRealtable) + { + QString errMsg = QObject::tr("Can not find log message table name!"); + TERRAMA2_LOG_ERROR() << errMsg; + throw terrama2::core::LogException() << ErrorDescription(errMsg); + } + + if(!tempDataSetRealtable->moveFirst()) + { + //CASE TABLE IS EMPTY + quantityRealtable = 0; + } + else + { + tempDataSetRealtable->moveFirst(); + quantityRealtable = tempDataSetRealtable->getInt32("pid_"+destinationDataSetName); + } + + if(quantityRealtable > quantityTableController || quantityRealtable < quantityTableController) + { + quantityTableController = quantityRealtable; + + boost::format queryInsertSize("UPDATE terrama2.table_ids_controller SET last_id = %1% WHERE table_name = '%2%'"); + queryInsertSize.bind_arg(1, quantityTableController); + queryInsertSize.bind_arg(2, destinationDataSetName); + + transactorDestination->execute(queryInsertSize.str()); + + transactorDestination->commit(); + } } std::string terrama2::core::DataStoragerTable::getGeometryPropertyName(DataSetPtr dataSet) const diff --git a/src/terrama2/impl/DataStoragerTable.hpp b/src/terrama2/impl/DataStoragerTable.hpp index 7d0efd284..d3b2bf5ab 100755 --- a/src/terrama2/impl/DataStoragerTable.hpp +++ b/src/terrama2/impl/DataStoragerTable.hpp @@ -65,6 +65,7 @@ namespace terrama2 std::shared_ptr updateAttributeNames(std::shared_ptr dataSet, std::shared_ptr dataSetType, DataSetPtr dataset) const; virtual std::string driver() const = 0; virtual std::string getStorageOption(DataSetPtr dataSet, DataSeriesPtr dataSetSeries) const; + void storeDynamicQuantity(std::shared_ptr transactorDestination, std::string destinationDataSetName) const; }; } } diff --git a/webapp/migrations/20200806000000-create-table-ids-controller.js b/webapp/migrations/20200806000000-create-table-ids-controller.js new file mode 100644 index 000000000..065fe097b --- /dev/null +++ b/webapp/migrations/20200806000000-create-table-ids-controller.js @@ -0,0 +1,17 @@ +module.exports = { + up: function(queryInterface, Sequelize) { + return queryInterface.createTable('table_ids_controller', { + id: { + type: Sequelize.INTEGER, + allowNull: false, + primaryKey: true, + autoIncrement: true + }, + table_name: Sequelize.STRING, + last_id: Sequelize.INTEGER + }, { schema: 'terrama2' }); + }, + down: function(queryInterface, /*Sequelize*/) { + return queryInterface.dropTable({ tableName: 'table_ids_controller', schema: 'terrama2' }); + } + }; \ No newline at end of file From ecf295334de96fe5d185f284cfaf1cd30bc33077 Mon Sep 17 00:00:00 2001 From: Juan Vinicius Date: Thu, 13 Aug 2020 16:40:48 -0300 Subject: [PATCH 2/4] Fix vector analysis bug (verify by id) --- .../vectorial_processing_intersection.sql | 24 +++--- src/terrama2/impl/DataStoragerTable.cpp | 77 ------------------ src/terrama2/impl/DataStoragerTable.hpp | 1 - .../analysis/core/AnalysisExecutor.cpp | 4 + .../core/vector-processing/Intersection.cpp | 79 +++++++++++++++++++ .../core/vector-processing/Intersection.hpp | 1 + .../core/vector-processing/Operator.hpp | 2 + 7 files changed, 95 insertions(+), 93 deletions(-) diff --git a/share/terrama2/scripts/sql-functions/vectorial_processing_intersection.sql b/share/terrama2/scripts/sql-functions/vectorial_processing_intersection.sql index a0f1366e6..10a3ce0b5 100644 --- a/share/terrama2/scripts/sql-functions/vectorial_processing_intersection.sql +++ b/share/terrama2/scripts/sql-functions/vectorial_processing_intersection.sql @@ -174,9 +174,9 @@ DECLARE number_of_attributes INTEGER; number_of_affected_rows BIGINT; - dynamic_last_id_inserted INTEGER; finaltable_last_id_inserted INTEGER; - is_finaltable_exists_idcontroller BOOLEAN; + is_dynamictable_info_exists BOOLEAN; + pk_output_table TEXT; query TEXT; result RECORD; @@ -201,24 +201,21 @@ BEGIN -- Getting date_column_from_dynamic_table EXECUTE format('SELECT get_date_column_from_dynamic_table(''%s'')', dynamic_table_name_handler) INTO date_column_from_dynamic_table; - -- Getting last id inserted in dynamic table - EXECUTE format('SELECT last_id FROM terrama2.table_ids_controller WHERE table_name = ''%s''', dynamic_table_name_handler) INTO dynamic_last_id_inserted; - final_table := format('%s_%s', output_table_name, analysis_id); - EXECUTE format('SELECT EXISTS (SELECT * FROM terrama2.table_ids_controller where table_name = ''%s'')', final_table) INTO is_finaltable_exists_idcontroller; + EXECUTE format('SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = ''public'' AND table_name = ''%s'')', output_table_name) INTO is_dynamictable_info_exists; - IF (is_finaltable_exists_idcontroller) THEN - EXECUTE format('SELECT last_id FROM terrama2.table_ids_controller WHERE table_name = ''%s''', final_table) INTO finaltable_last_id_inserted; + IF (is_dynamictable_info_exists) THEN + EXECUTE 'SELECT get_primary_key($1)' INTO pk_output_table USING output_table_name; + EXECUTE format('SELECT last_id FROM (SELECT * FROM public.%s ORDER BY %s DESC LIMIT 1) AS last_data',output_table_name,pk_output_table) INTO finaltable_last_id_inserted; ELSE - EXECUTE format('INSERT INTO terrama2.table_ids_controller (table_name, last_id) VALUES(''%s'', 0)', final_table); finaltable_last_id_inserted := 0; END IF; EXECUTE 'SELECT get_primary_key($1)' INTO pk_static_table USING static_table_name; EXECUTE 'SELECT get_primary_key($1)' INTO pk_dynamic_table USING dynamic_table_name_handler; - EXECUTE format('SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = ''public'' AND table_name = ''%s'')', final_table) INTO is_table_exists; + EXECUTE format('SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = ''public'' AND table_name = ''%s'')', final_table) INTO is_table_exists; EXECUTE format('SELECT COUNT(*) FROM information_schema.columns WHERE table_name = ''%s'' ', final_table) INTO number_of_columns; @@ -230,7 +227,7 @@ BEGIN -- 6 this is the default number of attributes: mo nitored_id, intersect_id, execution_date, intersection_geom, calculated_area_ha, final_table_id; IF (is_table_exists AND number_of_columns = number_of_attributes) THEN - analysis_filter := format(' %s.pid_%s > %s ', dynamic_table_name_handler, dynamic_table_name_handler, finaltable_last_id_inserted); + analysis_filter := format(' %s.%s > %s ', dynamic_table_name_handler, pk_dynamic_table, finaltable_last_id_inserted); date_filter := '1 = 1'; @@ -284,7 +281,7 @@ BEGIN GET DIAGNOSTICS number_of_affected_rows = ROW_COUNT; ELSE - analysis_filter := format(' %s.pid_%s > %s ', dynamic_table_name_handler, dynamic_table_name_handler, finaltable_last_id_inserted); + analysis_filter := format(' %s.%s > %s ', dynamic_table_name_handler, pk_dynamic_table, finaltable_last_id_inserted); date_filter := '1 = 1'; @@ -353,9 +350,6 @@ BEGIN EXECUTE query; END IF; - -- Refresh id number in ids controller - EXECUTE format('UPDATE terrama2.table_ids_controller SET last_id = %s WHERE table_name = ''%s''', dynamic_last_id_inserted, final_table); - query := format('ALTER TABLE %s ALTER COLUMN intersection_geom TYPE geometry(GEOMETRY, %s) USING ST_Transform(ST_SetSRID(intersection_geom,%s), %s)',final_table, dynamic_table_srid, dynamic_table_srid, dynamic_table_srid); EXECUTE query; diff --git a/src/terrama2/impl/DataStoragerTable.cpp b/src/terrama2/impl/DataStoragerTable.cpp index 2aa69cdb4..4d5284848 100755 --- a/src/terrama2/impl/DataStoragerTable.cpp +++ b/src/terrama2/impl/DataStoragerTable.cpp @@ -328,83 +328,6 @@ void terrama2::core::DataStoragerTable::store(DataSetSeries series, DataSetPtr o scopedTransaction.commit(); - if(this->dataSeries_.get()->semantics.temporality == terrama2::core::DataSeriesTemporality::DYNAMIC || this->dataSeries_.get()->semantics.dataSeriesType == terrama2::core::DataSeriesType::VECTOR_PROCESSING_OBJECT) - { - storeDynamicQuantity(transactorDestination, destinationDataSetName); - } -} - -void terrama2::core::DataStoragerTable::storeDynamicQuantity(std::shared_ptr transactorDestination, std::string destinationDataSetName) const -{ - - double quantityTableController; - - boost::format queryQuantityTableController("SELECT last_id from terrama2.table_ids_controller where table_name = '" + destinationDataSetName +"'"); - - std::shared_ptr tempDataSetQuantTableContr(transactorDestination->query(queryQuantityTableController.str())); - - if(!tempDataSetQuantTableContr) - { - QString errMsg = QObject::tr("Can not find log message table name!"); - TERRAMA2_LOG_ERROR() << errMsg; - throw terrama2::core::LogException() << ErrorDescription(errMsg); - } - - if(!tempDataSetQuantTableContr->moveFirst()) - { - //CASE TABLE IS EMPTY - quantityTableController = 0; - boost::format queryInsertInitial("INSERT INTO terrama2.table_ids_controller (table_name, last_id) VALUES('%1%', %2%)"); - queryInsertInitial.bind_arg(1, destinationDataSetName); - queryInsertInitial.bind_arg(2, 0); - - transactorDestination->execute(queryInsertInitial.str()); - - transactorDestination->commit(); - } - else - { - tempDataSetQuantTableContr->moveFirst(); - quantityTableController = tempDataSetQuantTableContr->getInt32("last_id"); - } - - double quantityRealtable; - - //RETRIEVE ID FROM LAST DATA INSERTED - boost::format queryRealTable("SELECT pid_"+destinationDataSetName+" FROM (SELECT * FROM public." + destinationDataSetName + " ORDER BY pid_" + destinationDataSetName + " DESC LIMIT 1) AS last_data"); - - std::shared_ptr tempDataSetRealtable(transactorDestination->query(queryRealTable.str())); - - if(!tempDataSetRealtable) - { - QString errMsg = QObject::tr("Can not find log message table name!"); - TERRAMA2_LOG_ERROR() << errMsg; - throw terrama2::core::LogException() << ErrorDescription(errMsg); - } - - if(!tempDataSetRealtable->moveFirst()) - { - //CASE TABLE IS EMPTY - quantityRealtable = 0; - } - else - { - tempDataSetRealtable->moveFirst(); - quantityRealtable = tempDataSetRealtable->getInt32("pid_"+destinationDataSetName); - } - - if(quantityRealtable > quantityTableController || quantityRealtable < quantityTableController) - { - quantityTableController = quantityRealtable; - - boost::format queryInsertSize("UPDATE terrama2.table_ids_controller SET last_id = %1% WHERE table_name = '%2%'"); - queryInsertSize.bind_arg(1, quantityTableController); - queryInsertSize.bind_arg(2, destinationDataSetName); - - transactorDestination->execute(queryInsertSize.str()); - - transactorDestination->commit(); - } } std::string terrama2::core::DataStoragerTable::getGeometryPropertyName(DataSetPtr dataSet) const diff --git a/src/terrama2/impl/DataStoragerTable.hpp b/src/terrama2/impl/DataStoragerTable.hpp index d3b2bf5ab..7d0efd284 100755 --- a/src/terrama2/impl/DataStoragerTable.hpp +++ b/src/terrama2/impl/DataStoragerTable.hpp @@ -65,7 +65,6 @@ namespace terrama2 std::shared_ptr updateAttributeNames(std::shared_ptr dataSet, std::shared_ptr dataSetType, DataSetPtr dataset) const; virtual std::string driver() const = 0; virtual std::string getStorageOption(DataSetPtr dataSet, DataSeriesPtr dataSetSeries) const; - void storeDynamicQuantity(std::shared_ptr transactorDestination, std::string destinationDataSetName) const; }; } } diff --git a/src/terrama2/services/analysis/core/AnalysisExecutor.cpp b/src/terrama2/services/analysis/core/AnalysisExecutor.cpp index 6573364bb..8eca063e5 100644 --- a/src/terrama2/services/analysis/core/AnalysisExecutor.cpp +++ b/src/terrama2/services/analysis/core/AnalysisExecutor.cpp @@ -509,8 +509,11 @@ void terrama2::services::analysis::core::AnalysisExecutor::runVectorialProcessin auto result = analysisOperator->getResultDataSet(); + auto lastId = analysisOperator->getLastId(); + context->addAttribute("table_name", te::dt::STRING_TYPE); context->addAttribute("affected_rows", te::dt::DOUBLE_TYPE); + context->addAttribute("last_id", te::dt::DOUBLE_TYPE); int currentLine = 0; @@ -520,6 +523,7 @@ void terrama2::services::analysis::core::AnalysisExecutor::runVectorialProcessin { context->setAnalysisResult(++currentLine, "table_name", result->getString("table_name")); context->setAnalysisResult(currentLine, "affected_rows", result->getDouble("affected_rows")); + context->setAnalysisResult(currentLine, "last_id", lastId); affectedRows = result->getDouble("affected_rows"); } diff --git a/src/terrama2/services/analysis/core/vector-processing/Intersection.cpp b/src/terrama2/services/analysis/core/vector-processing/Intersection.cpp index 89b493c3e..a00ea0128 100644 --- a/src/terrama2/services/analysis/core/vector-processing/Intersection.cpp +++ b/src/terrama2/services/analysis/core/vector-processing/Intersection.cpp @@ -1,8 +1,13 @@ #include "Intersection.hpp" #include "../../../../core/utility/Utils.hpp" +#include "../../../../core/utility/Raii.hpp" #include "../Analysis.hpp" +//Boost +#include +#include + #include terrama2::services::analysis::core::vp::Intersection::Intersection(terrama2::services::analysis::core::AnalysisPtr analysis, @@ -20,6 +25,80 @@ terrama2::services::analysis::core::vp::Intersection::~Intersection() { } +double terrama2::services::analysis::core::vp::Intersection::getLastId() +{ + auto transactor = dataSource_->getTransactor(); + std::string dynamicLayerTableName = terrama2::core::getTableNameProperty(dynamicDataSeries_->datasetList[0]); + + //VARIFY PK FROM TABLE + std::string columnPk = ""; + + std::string queryColumnPkStr = "SELECT kcu.column_name FROM INFORMATION_SCHEMA.TABLES t "; + queryColumnPkStr = queryColumnPkStr + "LEFT JOIN INFORMATION_SCHEMA.TABLE_CONSTRAINTS tc "; + queryColumnPkStr = queryColumnPkStr + "ON tc.table_catalog = t.table_catalog "; + queryColumnPkStr = queryColumnPkStr + "AND tc.table_schema = t.table_schema "; + queryColumnPkStr = queryColumnPkStr + "AND tc.table_name = t.table_name "; + queryColumnPkStr = queryColumnPkStr + "AND tc.constraint_type = 'PRIMARY KEY' "; + queryColumnPkStr = queryColumnPkStr + "LEFT JOIN INFORMATION_SCHEMA.KEY_COLUMN_USAGE kcu "; + queryColumnPkStr = queryColumnPkStr + "ON kcu.table_catalog = tc.table_catalog "; + queryColumnPkStr = queryColumnPkStr + "AND kcu.table_schema = tc.table_schema "; + queryColumnPkStr = queryColumnPkStr + "AND kcu.table_name = tc.table_name "; + queryColumnPkStr = queryColumnPkStr + "AND kcu.constraint_name = tc.constraint_name "; + queryColumnPkStr = queryColumnPkStr + "WHERE t.table_schema NOT IN ('pg_catalog', 'information_schema') and t.table_name = '"+dynamicLayerTableName+"'"; + + std::cout << queryColumnPkStr << std::endl; + + boost::format queryColumnPk(queryColumnPkStr); + + std::shared_ptr tempDataSetColumnPk(transactor->query(queryColumnPk.str())); + + if(!tempDataSetColumnPk) + { + QString errMsg = QObject::tr("Can not find log message table name!"); + TERRAMA2_LOG_ERROR() << errMsg; + throw terrama2::core::LogException() << ErrorDescription(errMsg); + } + + if(!tempDataSetColumnPk->moveFirst()) + { + QString errMsg = QObject::tr("Can not find column PK!"); + TERRAMA2_LOG_ERROR() << errMsg; + throw terrama2::core::LogException() << ErrorDescription(errMsg); + } + else + { + tempDataSetColumnPk->moveFirst(); + columnPk = tempDataSetColumnPk->getString("column_name"); + } + + double quantityRealtable; + + //RETRIEVE ID FROM LAST DATA INSERTED + boost::format queryRealTable("SELECT "+columnPk+" FROM (SELECT * FROM public." + dynamicLayerTableName + " ORDER BY " + columnPk + " DESC LIMIT 1) AS last_data"); + + std::shared_ptr tempDataSetRealtable(transactor->query(queryRealTable.str())); + + if(!tempDataSetRealtable) + { + QString errMsg = QObject::tr("Can not find log message table name!"); + TERRAMA2_LOG_ERROR() << errMsg; + throw terrama2::core::LogException() << ErrorDescription(errMsg); + } + + if(!tempDataSetRealtable->moveFirst()) + { + //CASE TABLE IS EMPTY + quantityRealtable = 0; + } + else + { + tempDataSetRealtable->moveFirst(); + quantityRealtable = tempDataSetRealtable->getInt32(columnPk); + } + + return quantityRealtable; +} + void terrama2::services::analysis::core::vp::Intersection::execute() { auto transactor = dataSource_->getTransactor(); diff --git a/src/terrama2/services/analysis/core/vector-processing/Intersection.hpp b/src/terrama2/services/analysis/core/vector-processing/Intersection.hpp index 1b71dc5bd..dd479c79f 100644 --- a/src/terrama2/services/analysis/core/vector-processing/Intersection.hpp +++ b/src/terrama2/services/analysis/core/vector-processing/Intersection.hpp @@ -35,6 +35,7 @@ namespace terrama2 ~Intersection() override; virtual void execute() override; + virtual double getLastId() override; }; } } diff --git a/src/terrama2/services/analysis/core/vector-processing/Operator.hpp b/src/terrama2/services/analysis/core/vector-processing/Operator.hpp index a4f51aa75..df69cb514 100644 --- a/src/terrama2/services/analysis/core/vector-processing/Operator.hpp +++ b/src/terrama2/services/analysis/core/vector-processing/Operator.hpp @@ -49,6 +49,8 @@ namespace terrama2 */ virtual void execute() = 0; + virtual double getLastId() = 0; + virtual void setWhereCondition(const std::string& where); virtual std::shared_ptr getResultDataSet() const; From 00b833d4c1ed5cb4403fd59b64e92bb4c71fca57 Mon Sep 17 00:00:00 2001 From: Juan Vinicius Date: Mon, 17 Aug 2020 10:23:45 -0300 Subject: [PATCH 3/4] verify analysis table name --- .../core/vector-processing/Intersection.cpp | 47 ++++++++++++++++++- .../core/vector-processing/Intersection.hpp | 1 + 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/src/terrama2/services/analysis/core/vector-processing/Intersection.cpp b/src/terrama2/services/analysis/core/vector-processing/Intersection.cpp index a00ea0128..4d51ab137 100644 --- a/src/terrama2/services/analysis/core/vector-processing/Intersection.cpp +++ b/src/terrama2/services/analysis/core/vector-processing/Intersection.cpp @@ -28,7 +28,7 @@ terrama2::services::analysis::core::vp::Intersection::~Intersection() double terrama2::services::analysis::core::vp::Intersection::getLastId() { auto transactor = dataSource_->getTransactor(); - std::string dynamicLayerTableName = terrama2::core::getTableNameProperty(dynamicDataSeries_->datasetList[0]); + std::string dynamicLayerTableName = getDynamicTableName(); //VARIFY PK FROM TABLE std::string columnPk = ""; @@ -99,6 +99,51 @@ double terrama2::services::analysis::core::vp::Intersection::getLastId() return quantityRealtable; } +std::string terrama2::services::analysis::core::vp::Intersection::getDynamicTableName() +{ + auto transactor = dataSource_->getTransactor(); + std::string dynamicLayerTableName = terrama2::core::getTableNameProperty(dynamicDataSeries_->datasetList[0]); + std::string tableNameReturn = ""; + + boost::format queryVerifyTable("SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS WHERE table_name = '"+dynamicLayerTableName+"' AND column_name = 'table_name'"); + + std::shared_ptr tempDataSetVerifyTable(transactor->query(queryVerifyTable.str())); + + if(!tempDataSetVerifyTable) + { + QString errMsg = QObject::tr("Can not find!"); + TERRAMA2_LOG_ERROR() << errMsg; + throw terrama2::core::LogException() << ErrorDescription(errMsg); + } + + tempDataSetVerifyTable->moveFirst(); + auto verifyTableResult = tempDataSetVerifyTable->getInt64("count"); + + if(verifyTableResult > 0) + { + boost::format queryGetRealName("SELECT table_name FROM public."+dynamicLayerTableName); + + std::shared_ptr tempDataSetRealName(transactor->query(queryGetRealName.str())); + + if(!tempDataSetRealName) + { + QString errMsg = QObject::tr("Can not find!"); + TERRAMA2_LOG_ERROR() << errMsg; + throw terrama2::core::LogException() << ErrorDescription(errMsg); + } + + tempDataSetRealName->moveFirst(); + tableNameReturn = tempDataSetRealName->getString("table_name"); + } + else + { + tableNameReturn = dynamicLayerTableName; + } + + return tableNameReturn; + +} + void terrama2::services::analysis::core::vp::Intersection::execute() { auto transactor = dataSource_->getTransactor(); diff --git a/src/terrama2/services/analysis/core/vector-processing/Intersection.hpp b/src/terrama2/services/analysis/core/vector-processing/Intersection.hpp index dd479c79f..fea40fab2 100644 --- a/src/terrama2/services/analysis/core/vector-processing/Intersection.hpp +++ b/src/terrama2/services/analysis/core/vector-processing/Intersection.hpp @@ -36,6 +36,7 @@ namespace terrama2 virtual void execute() override; virtual double getLastId() override; + virtual std::string getDynamicTableName(); }; } } From 809409c9ab313704fe4d3fff9d9d0df5a3a65ad1 Mon Sep 17 00:00:00 2001 From: Juan Vinicius Date: Tue, 25 Aug 2020 09:43:00 -0300 Subject: [PATCH 4/4] Grid Sample accept historic date --- src/terrama2/core/utility/FilterUtils.cpp | 166 ++++++++++++++++++ src/terrama2/core/utility/FilterUtils.hpp | 4 + .../services/analysis/core/grid/Operator.cpp | 19 +- .../services/analysis/core/grid/Operator.hpp | 2 +- .../core/python/PythonBindingGrid.cpp | 4 +- .../angular/analysis/data/grid-operators.json | 2 +- 6 files changed, 190 insertions(+), 7 deletions(-) diff --git a/src/terrama2/core/utility/FilterUtils.cpp b/src/terrama2/core/utility/FilterUtils.cpp index 2371177b7..f975cc136 100644 --- a/src/terrama2/core/utility/FilterUtils.cpp +++ b/src/terrama2/core/utility/FilterUtils.cpp @@ -38,6 +38,9 @@ #include #include +//Terralib +#include + // Qt #include #include @@ -46,6 +49,8 @@ #include #include +#include + // TODO: This method is being call for every check of the mask, this should only be called once to avoid a costy operation. std::string terrama2::core::terramaMask2Regex(const std::string& mask) { @@ -182,6 +187,167 @@ std::shared_ptr terrama2::core::getFileTimestamp(const st return timeStamp; } +int terrama2::core::getMaxDay(int month, int year) { + if(month == 0 || month == 2 || month == 4 || month == 6 || month == 7 || month == 9 || month == 11) + return 31; + else if(month == 3 || month == 5 || month == 8 || month == 10) + return 30; + else { + if(year % 4 == 0) { + if(year % 100 == 0) { + if(year % 400 == 0) + return 29; + return 28; + } + return 29; + } + return 28; + } +} + +std::shared_ptr terrama2::core::gridSampleVerifyMask(std::string mask, std::shared_ptr actualDate) +{ + std::shared_ptr dateReturn; + auto temp = QString::fromStdString(mask); + auto list = temp.split("%", QString::SkipEmptyParts); + + time_t theTime = time(NULL); + struct tm *aTime = localtime(&theTime); + + int day = aTime->tm_mday; + int month = aTime->tm_mon + 1; // Month is 0 - 11, add 1 to get a jan-dec 1-12 concept + int year = aTime->tm_year + 1900; // Year is # years since 1900 + + std::vector options; + options.push_back("YYYY"); + options.push_back("MM"); + options.push_back("DD"); + options.push_back("hh"); + options.push_back("mm"); + options.push_back("ss"); + + int optIndice = 0; + + std::map mapDate; + + for(int i = 0; i < list.size(); ++i) + { + auto& parts = list[i]; + auto listParts = parts.split("$", QString::SkipEmptyParts); + + auto& part = listParts[0]; + std::string partStr = part.toUtf8().constData(); + + auto& partContent = listParts[1]; + std::string partContentStr = partContent.toUtf8().constData(); + + for(int y=optIndice; y < options.size(); ++y) + { + if(partStr.compare(options[y]) == 0) + { + switch (y) { + case 0: + mapDate.insert({"year", partContentStr}); + break; + case 1: + mapDate.insert({"month", partContentStr}); + break; + case 2: + mapDate.insert({"day", partContentStr}); + break; + case 3: + mapDate.insert({"hour", partContentStr}); + break; + case 4: + mapDate.insert({"minute", partContentStr}); + break; + case 5: + mapDate.insert({"second", partContentStr}); + break; + default: + std::cout << "DEFAULT" << std::endl; + } + + optIndice++; + if((i+1) == list.size()) + { + continue; + } + break; + } + else + { + int yearAux = 0; + int monthAux = 0; + switch (y) { + case 0: + mapDate.insert({"year", std::to_string(year)}); + break; + case 1: + if(month < 10) + { + mapDate.insert({"month", "0"+std::to_string(month)}); + } + else + { + mapDate.insert({"month", std::to_string(month)}); + } + break; + case 2: + yearAux = std::stoi(mapDate.find("year")->second); + monthAux = std::stoi(mapDate.find("month")->second) - 1; + day = getMaxDay(monthAux,yearAux); + mapDate.insert({"day", std::to_string(day)}); + break; + case 3: + mapDate.insert({"hour", "00"}); + break; + case 4: + mapDate.insert({"minute", "00"}); + break; + case 5: + mapDate.insert({"second", "00"}); + break; + default: + std::cout << "DEFAULT" << std::endl; + } + optIndice++; + } + } + } + + if(!mapDate.empty()) + { + + auto mapyear = mapDate.find("year"); + auto mapmonth = mapDate.find("month"); + auto mapday = mapDate.find("day"); + auto maphour = mapDate.find("hour"); + auto mapminute = mapDate.find("minute"); + auto mapsecond = mapDate.find("second"); + + std::string finalDate = mapyear->second+"-"; + finalDate = finalDate + mapmonth->second+"-"; + finalDate = finalDate + mapday->second; + finalDate = finalDate + "T"; + finalDate = finalDate + maphour->second+":"; + finalDate = finalDate + mapminute->second+":"; + finalDate = finalDate + mapsecond->second; + finalDate = finalDate + "-00"; + + dateReturn = terrama2::core::TimeUtils::stringToTimestamp(finalDate, terrama2::core::TimeUtils::webgui_timefacet); + + return dateReturn; + } + else + { + QString errMsg = QObject::tr("The mask don't have the minimal needed parameters of a date!"); + TERRAMA2_LOG_ERROR() << errMsg; + throw terrama2::core::UtilityException() << ErrorDescription(errMsg); + } + return nullptr; +} + bool terrama2::core::isValidDataSetName(const std::string& mask, const Filter& filter, const std::string& timezone, diff --git a/src/terrama2/core/utility/FilterUtils.hpp b/src/terrama2/core/utility/FilterUtils.hpp index e28cf7084..f6ae3d97b 100644 --- a/src/terrama2/core/utility/FilterUtils.hpp +++ b/src/terrama2/core/utility/FilterUtils.hpp @@ -121,6 +121,10 @@ namespace terrama2 * @return Regex style mask */ TMCOREEXPORT std::string terramaMask2Regex(const std::string& mask); + + TMCOREEXPORT std::shared_ptr gridSampleVerifyMask(std::string mask, std::shared_ptr actualDate); + + TMCOREEXPORT int getMaxDay(int month, int year); } // end namespace core } // end namespace terrama2 diff --git a/src/terrama2/services/analysis/core/grid/Operator.cpp b/src/terrama2/services/analysis/core/grid/Operator.cpp index fb938ac4f..6673b4f29 100644 --- a/src/terrama2/services/analysis/core/grid/Operator.cpp +++ b/src/terrama2/services/analysis/core/grid/Operator.cpp @@ -31,6 +31,7 @@ #include "../ContextManager.hpp" #include "../../../../core/data-model/DataSetGrid.hpp" #include "../../../../core/utility/Logger.hpp" +#include "../../../../core/utility/FilterUtils.hpp" #include "../utility/Utils.hpp" #include "../utility/Verify.hpp" #include "../GridContext.hpp" @@ -59,12 +60,12 @@ double terrama2::services::analysis::core::grid::getValue(std::shared_ptrgetStartTime(); - filter.discardAfter = startTime; + if(date.compare("") != 0) + { + filter.discardAfter = terrama2::core::gridSampleVerifyMask(date, context->getStartTime()); + } + else + { + filter.discardAfter = startTime; + } + filter.lastValues = std::make_shared(1); filter.isReprocessingHistoricalData = true; } else { + if(date.compare("") != 0) + { + filter.discardAfter = terrama2::core::gridSampleVerifyMask(date, context->getStartTime()); + } filter.lastValues = std::make_shared(1); } diff --git a/src/terrama2/services/analysis/core/grid/Operator.hpp b/src/terrama2/services/analysis/core/grid/Operator.hpp index 4534f5261..d87aceba3 100644 --- a/src/terrama2/services/analysis/core/grid/Operator.hpp +++ b/src/terrama2/services/analysis/core/grid/Operator.hpp @@ -56,7 +56,7 @@ namespace terrama2 \param dataSeriesName DataSeries name. \return The current pixel value for the selected data series. */ - TMANALYSISEXPORT double sample(const std::string& dataSeriesName, size_t bandIdx = 0); + TMANALYSISEXPORT double sample(const std::string& dataSeriesName, size_t bandIdx = 0, const std::string& date=""); TMANALYSISEXPORT double getValue(std::shared_ptr raster, std::shared_ptr interpolator, double column, double row, size_t bandIdx); diff --git a/src/terrama2/services/analysis/core/python/PythonBindingGrid.cpp b/src/terrama2/services/analysis/core/python/PythonBindingGrid.cpp index 6de286132..fb0cd5c4c 100644 --- a/src/terrama2/services/analysis/core/python/PythonBindingGrid.cpp +++ b/src/terrama2/services/analysis/core/python/PythonBindingGrid.cpp @@ -63,7 +63,7 @@ void terrama2::services::analysis::core::python::Grid::registerFunctions() #pragma GCC diagnostic ignored "-Wunused-local-typedef" // // Declaration needed for default parameter restriction -BOOST_PYTHON_FUNCTION_OVERLOADS(gridSample_overloads, terrama2::services::analysis::core::grid::sample, 1, 2) +BOOST_PYTHON_FUNCTION_OVERLOADS(gridSample_overloads, terrama2::services::analysis::core::grid::sample, 1, 3) // closing "-Wunused-local-typedef" pragma #pragma GCC diagnostic pop @@ -81,7 +81,7 @@ void terrama2::services::analysis::core::python::Grid::registerGridFunctions() scope gridScope = gridModule; // export functions inside grid namespace - def("sample", terrama2::services::analysis::core::grid::sample, gridSample_overloads(args("dataSeriesName", "band"), "Grid sample operator.")); + def("sample", terrama2::services::analysis::core::grid::sample, gridSample_overloads(args("dataSeriesName", "band", "date"), "Grid sample operator.")); } // pragma to silence python macros warnings diff --git a/webapp/public/javascripts/angular/analysis/data/grid-operators.json b/webapp/public/javascripts/angular/analysis/data/grid-operators.json index 56efeb6a5..37d6b94e2 100644 --- a/webapp/public/javascripts/angular/analysis/data/grid-operators.json +++ b/webapp/public/javascripts/angular/analysis/data/grid-operators.json @@ -2,6 +2,6 @@ { "name": "Sample", "description": "Retrieves a cell value based in interpolation method to grid analysis", - "code": "grid.sample(\"\", [])" + "code": "grid.sample(\"\", [], [\"\"])" } ]