Squashed 'seims/src/ccgl/' changes from 7441ea46..c3cb5566

c3cb5566 Update download link of GDAL libarary on Windows 5772bc58 (performance improved)Add valid position index (1D array, pos_idx_) and will remove pos_data_ in next version. 809ed126 Merge branch 'dev' abf08c78 feat: sync SEIMS to CCGL, and add `opts` to filter metadata of files in mongodb ce81cfe2 Merge branch 'dev' 9cbf53c1 disable jekyll git-subtree-dir: seims/src/ccgl git-subtree-split: c3cb5566bcafec3efb41b579d6df50e881a109ee
lreis2415 · Jul 15, 2023 · cddc87b · cddc87b
1 parent db18923
commit cddc87b
Show file tree

Hide file tree

Showing 17 changed files with 351 additions and 77 deletions.
diff --git a/.github/workflows/cmake_builds_with-gdal-mongodb.yml b/.github/workflows/cmake_builds_with-gdal-mongodb.yml
@@ -93,7 +93,6 @@ jobs:
       with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           commit_message: ${{ github.event.head_commit.message }}
-          disable_nojekyll: true
           publish_branch: gh-pages
           force_orphan: true
           publish_dir: html

diff --git a/.github/workflows/download_gdal.ps1 b/.github/workflows/download_gdal.ps1
@@ -1,8 +1,8 @@
 param ($gdalPath = "$env:SystemDrive\gdal", $VSversion = "1928", $GDALversion = "3.5.3", $MAPSversion = "8.0.0")
 $GDALversion=$GDALversion -replace '\.','-'
 $MAPSversion=$MAPSversion -replace '\.','-'
-$urllib = "https://build2.gisinternals.com/sdk/downloads/release-$VSversion-x64-gdal-$GDALversion-mapserver-$MAPSversion-libs.zip"
-$urlbin = "https://build2.gisinternals.com/sdk/downloads/release-$VSversion-x64-gdal-$GDALversion-mapserver-$MAPSversion.zip"
+$urllib = "https://download.gisinternals.com/sdk/downloads/release-$VSversion-x64-gdal-$GDALversion-mapserver-$MAPSversion-libs.zip"
+$urlbin = "https://download.gisinternals.com/sdk/downloads/release-$VSversion-x64-gdal-$GDALversion-mapserver-$MAPSversion.zip"
 $zipLibFile = "$gdalPath\gdallib.zip"
 $zipBinFile = "$gdalPath\gdal.zip"
 

diff --git a/src/data_raster.cpp b/src/data_raster.cpp
@@ -198,6 +198,7 @@ bool SubsetPositions::Initialization() {
     g_ecol = -1;
     alloc_ = false;
     local_pos_ = nullptr;
+    local_posidx_ = nullptr;
     global_ = nullptr;
     data_ = nullptr;
     data2d_ = nullptr;
@@ -229,6 +230,7 @@ SubsetPositions::SubsetPositions(SubsetPositions*& src, const bool deep_copy) {
         alloc_ = true;
         Initialize1DArray(n_cells, global_, src->global_);
         Initialize2DArray(n_cells, 2, local_pos_, src->local_pos_);
+        Initialize1DArray(n_cells, local_posidx_, src->local_posidx_);
         if (nullptr != src->data_) {
             Initialize1DArray(n_cells, data_, src->data_);
         }
@@ -240,6 +242,7 @@ SubsetPositions::SubsetPositions(SubsetPositions*& src, const bool deep_copy) {
         alloc_ = false;
         global_ = src->global_;
         local_pos_ = src->local_pos_;
+        local_posidx_ = src->local_posidx_;
         if (nullptr != src->data_) { data_ = src->data_; }
         if (nullptr != src->data2d_) { data2d_ = src->data2d_; }
     }
@@ -250,6 +253,10 @@ SubsetPositions::~SubsetPositions() {
         if (alloc_) { Release2DArray(local_pos_); }
         else { local_pos_ = nullptr; }
     }
+    if (nullptr != local_posidx_) {
+        if (alloc_) { Release1DArray(local_posidx_); }
+        else { local_posidx_ = nullptr; }
+    }
     if (nullptr != global_) {
         if (alloc_) { Release1DArray(global_); }
         else { global_ = nullptr; }
@@ -282,7 +289,8 @@ bool SubsetPositions::ReadFromMongoDB(MongoGridFs* gfs, const string& fname, con
             Initialize1DArray(n_cells, data_, NODATA_VALUE);
         }
         for (int i = 0; i < n_cells; i++) {
-            data_[i] = dbdata[local_pos_[i][0] * ncols + local_pos_[i][1]];
+            //data_[i] = dbdata[local_pos_[i][0] * ncols + local_pos_[i][1]];
+            data_[i] = dbdata[local_posidx_[i]];
         }
     }
     else {
@@ -292,7 +300,8 @@ bool SubsetPositions::ReadFromMongoDB(MongoGridFs* gfs, const string& fname, con
         for (int i = 0; i < n_cells; i++) {
             for (int j = 0; j < n_lyrs; j++) {
                 if (nfull == db_ncells) { // consider data from MongoDB is fullsize data
-                    data2d_[i][j] = dbdata[(local_pos_[i][0] * ncols + local_pos_[i][1]) * n_lyrs + j];
+                    //data2d_[i][j] = dbdata[(local_pos_[i][0] * ncols + local_pos_[i][1]) * n_lyrs + j];
+                    data2d_[i][j] = dbdata[local_posidx_[i] * n_lyrs + j];
                 }
                 else { data2d_[i][j] = dbdata[i * n_lyrs + j]; }
             }

diff --git a/src/data_raster.hpp b/src/data_raster.hpp
diff --git a/src/db_mongoc.cpp b/src/db_mongoc.cpp
@@ -239,7 +239,7 @@ void MongoDatabase::GetCollectionNames(vector<string>& collnames) {
                 // Refers to https://stackoverflow.com/a/18703743/4837280
                 std::for_each(splitstrs.begin(), splitstrs.end() - 1,
                               [&](const std::string &piece){ realname += piece; });
-                string another_coll = splitstrs.back() == "files" ? 
+                string another_coll = splitstrs.back() == "files" ?
                                           realname + ".chunks" :
                                           realname + ".files";
                 if (find(collnames.begin(), collnames.end(), another_coll) == collnames.end()) {
@@ -278,7 +278,7 @@ mongoc_cursor_t* MongoCollection::ExecuteQuery(const bson_t* b) {
     //       Upd 12/13/2017 The new method also failed in our linux cluster (redhat 6.2 and Intel C++ 12.1).
     //       Upd 12/29/2021 I decide to use new method from a quite later version such as v1.8.0.
     //                      Maybe a precise version can be determined after a thorough test.
-    //       Upd 06/24/2022 The new API still not working in Windows. 
+    //       Upd 06/24/2022 The new API still not working in Windows.
     mongoc_cursor_t* cursor = nullptr;
 // #if MONGOC_CHECK_VERSION(1, 8, 0)
 //     cursor = mongoc_collection_find_with_opts(collection_, b, NULL, NULL);
@@ -420,13 +420,17 @@ bson_t* MongoGridFs::GetFileMetadata(string const& gfilename,
 
 bool MongoGridFs::GetStreamData(string const& gfilename, char*& databuf,
                                 vint& datalength, mongoc_gridfs_t* gfs /* = NULL */,
-                                STRING_MAP opts /* = STRING_MAP() */) {
+                                const STRING_MAP* opts /* = nullptr */) {
     if (gfs_ != NULL) { gfs = gfs_; }
     if (NULL == gfs) {
         StatusMessage("mongoc_gridfs_t must be provided for MongoGridFs!");
         return false;
     }
-    mongoc_gridfs_file_t* gfile = GetFile(gfilename, gfs, opts);
+    STRING_MAP opts_temp;
+    if (nullptr == opts) {
+        opts = &opts_temp;
+    }
+    mongoc_gridfs_file_t* gfile = GetFile(gfilename, gfs, *opts);
     if (NULL == gfile) {
         databuf = NULL;
         StatusMessage(("MongoGridFs::GetStreamData(" + gfilename + ") failed!").c_str());
@@ -564,7 +568,10 @@ bool GetBoolFromBson(bson_t* bmeta, const char* key) {
 time_t GetDatetimeFromBsonIterator(bson_iter_t* iter) {
     const bson_value_t* vv = bson_iter_value(iter);
     if (vv->value_type == BSON_TYPE_DATE_TIME) {
-        return CVT_TIMET(vv->value.v_datetime);
+		string str_milisec = std::to_string(vv->value.v_datetime);
+		string str_second = str_milisec.substr(0, str_milisec.length() - 3);
+		int intStr = atoi(str_second.c_str());
+        return CVT_TIMET(intStr);
     }
     if (vv->value_type == BSON_TYPE_UTF8) {
         string tmp_time_str = vv->value.v_utf8.str;

diff --git a/src/db_mongoc.h b/src/db_mongoc.h
@@ -167,7 +167,7 @@ class MongoGridFs {
     /*! Get stream data of a given GridFS file name */
     bool GetStreamData(string const& gfilename, char*& databuf, vint& datalength,
                        mongoc_gridfs_t* gfs = NULL,
-                       STRING_MAP opts = STRING_MAP());
+                       const STRING_MAP* opts = nullptr);
 
     /*! Write stream data to a GridFS file */
     bool WriteStreamData(const string& gfilename, char*& buf, vint length,

diff --git a/src/utils_array.h b/src/utils_array.h
@@ -50,6 +50,8 @@ bool Initialize1DArray(int row, T*& data, INI_T init_value);
 template <typename T, typename INI_T>
 bool Initialize1DArray(int row, T*& data, INI_T* init_data);
 
+template <typename T, typename INI_T>
+bool Initialize1DArray4ItpWeight(int row, T*& data, INI_T* init_data, int itp_weight_data_length);
 /*!
  * \brief Initialize DT_Array2D data
  *
@@ -329,11 +331,12 @@ class Array2D {
 template <typename T, typename INI_T>
 bool Initialize1DArray(const int row, T*& data, const INI_T init_value) {
     if (nullptr != data) {
-        cout << "The input 1D array pointer is not nullptr, without initialized!" << endl;
+        //Should allow an array to re-enter this function then just return? --wyj
+        //cout << "The input 1D array pointer is not nullptr. No initialization performed!" << endl;
         return false;
     }
     if (row <= 0) {
-        cout << "The data length MUST greater than 0!" << endl;
+        cout << "The data length MUST be greater than 0!" << endl;
         data = nullptr;
         return false;
     }
@@ -355,7 +358,7 @@ bool Initialize1DArray(const int row, T*& data, const INI_T init_value) {
 template <typename T, typename INI_T>
 bool Initialize1DArray(const int row, T*& data, INI_T* const init_data) {
     if (nullptr != data) {
-        cout << "The input 1D array pointer is not nullptr, without initialized!" << endl;
+        cout << "The input 1D array pointer is not nullptr. No initialization performed!" << endl;
         return false;
     }
     data = new(nothrow) T[row];
@@ -376,13 +379,10 @@ bool Initialize1DArray(const int row, T*& data, INI_T* const init_data) {
 }
 
 template <typename T, typename INI_T>
-bool Initialize2DArray(const int row, const int col, T**& data, const INI_T init_value) {
-    if (row <= 0 || col <= 0) {
-        cout << "The row and col should not be less or equal to ZERO!" << endl;
-        return false;
-    }
+bool Initialize2DArray(const int row, const int col, T**& data,
+                       const INI_T init_value) {
     if (nullptr != data) {
-        cout << "The input 2D array pointer is not nullptr, without initialized!" << endl;
+        cout << "The input 2D array pointer is not nullptr. No initialization performed!" << endl;
         return false;
     }
     data = new(nothrow) T*[row];

diff --git a/src/utils_time.cpp b/src/utils_time.cpp
@@ -72,6 +72,43 @@ string ConvertToString2(const time_t date, const bool utc_time /* = true */) {
     delete date_info;
     return s;
 }
+/// format: 2022_11_17_092000
+string ConvertToString3(const time_t date, const bool utc_time /* = true */) {
+	static int month_days[13] = { 0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 };
+	struct tm* date_info = new tm();
+	GetDateTime(date, date_info, utc_time);
+	if (date_info->tm_isdst > 0) {
+		if (date_info->tm_hour != 0) {
+			date_info->tm_hour -= 1;
+		}
+		else {
+			date_info->tm_hour = 23;
+			date_info->tm_mday -= 1;
+			if (date_info->tm_mday == 0) {
+				date_info->tm_mon -= 1;
+
+				if (date_info->tm_mon == 0) {
+					date_info->tm_year -= 1;
+					date_info->tm_mon = 12;
+					date_info->tm_mday = 31;
+				}
+				else {
+					if (IsLeapYear(date_info->tm_year)) {
+						date_info->tm_mday = month_days[date_info->tm_mon] + 1;
+					}
+					else {
+						date_info->tm_mday = month_days[date_info->tm_mon];
+					}
+				}
+			}
+		}
+	}
+	char date_string[30];
+	strftime(date_string, 30, "%Y_%m_%d_%H%M%S", date_info);
+	string s(date_string);
+	delete date_info;
+	return s;
+}
 
 time_t ConvertToTime(const string& str_date, string const& format, const bool include_hour,
                      const bool utc_time /* = true */) {

diff --git a/src/utils_time.h b/src/utils_time.h
@@ -45,7 +45,13 @@ string ConvertToString(const time_t date, bool utc_time = true);
  * \return Date time \a string
  */
 string ConvertToString2(const time_t date, bool utc_time = true);
-
+/*!
+ * \brief Convert date time to string as the format of "YYYY_MM_DD_HHMMSS"
+ * \param[in] date \a time_t data type
+ * \param[in] utc_time By default, the input date is under UTC+00:00 timezone.
+ * \return Date time \a string
+ */
+string ConvertToString3(const time_t date, bool utc_time  = true);
 /*!
  * \brief Convert string to date time, string format could be %4d%2d%2d or %d-%d-%d
  *