Merge release/2.25.0 to master

.appveyor.yml

@@ -2,7 +2,7 @@
 #      general configuration      #
 #---------------------------------#
 
-version: 2.24.3-{build}-{branch}
+version: 2.25.0-{build}-{branch}
 
 branches:
   only:
@@ -74,6 +74,7 @@ build_script:
                -D ENABLE_FORTRAN=0 ^
                -D ENABLE_NETCDF=1 ^
                -D ENABLE_JPG=1 ^
+               -D ENABLE_AEC=0 ^
                -D IEEE_LE=1 ^
                -D ENABLE_MEMFS=1 ^
                -D ENABLE_EXTRA_TESTS=ON ^

.github/workflows/ci.yml

@@ -27,5 +27,7 @@ jobs:
       codecov_upload: true
       build_package_inputs: |
         self_coverage: true
-        dependencies: ecmwf/ecbuild
+        dependencies: |
+          ecmwf/ecbuild
+          MathisRosenhauer/libaec@master
         dependency_branch: develop

CMakeLists.txt

@@ -21,27 +21,29 @@ cmake_minimum_required( VERSION 3.12 FATAL_ERROR )
 find_package( ecbuild 3.4 REQUIRED HINTS ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../ecbuild)
 
 # Initialise project
-project( eccodes VERSION 2.24.3 LANGUAGES C )
+project( eccodes VERSION 2.25.0 LANGUAGES C )
 
 ###############################################################################
 # system checks needed for eccodes_config.h and some options like MEMFS
 
-check_type_size( int            ECCODES_SIZEOF_INT )
-check_type_size( long           ECCODES_SIZEOF_LONG )
-check_type_size( size_t         ECCODES_SIZEOF_SIZE_T )
-
-check_include_files( assert.h         ECCODES_HAVE_ASSERT_H )
-check_include_files( string.h         ECCODES_HAVE_STRING_H )
-check_include_files( sys/types.h      ECCODES_HAVE_SYS_TYPES_H )
-check_include_files( sys/stat.h       ECCODES_HAVE_SYS_STAT_H )
-check_include_files( fcntl.h          ECCODES_HAVE_FCNTL_H )
-check_include_files( unistd.h         ECCODES_HAVE_UNISTD_H )
-
-check_symbol_exists( fseeko           "stdio.h"    ECCODES_HAVE_FSEEKO )
-check_symbol_exists( posix_memalign   "stdlib.h"   ECCODES_HAVE_POSIX_MEMALIGN )
-check_symbol_exists( fmemopen         "stdio.h"    ECCODES_HAVE_FMEMOPEN )
-check_symbol_exists( funopen          "stdio.h"    ECCODES_HAVE_FUNOPEN )
-check_symbol_exists( realpath         "stdlib.h"   ECCODES_HAVE_REALPATH )
+check_type_size( int     ECCODES_SIZEOF_INT )
+check_type_size( long    ECCODES_SIZEOF_LONG )
+check_type_size( size_t  ECCODES_SIZEOF_SIZE_T )
+
+check_include_files( assert.h      ECCODES_HAVE_ASSERT_H )
+check_include_files( string.h      ECCODES_HAVE_STRING_H )
+check_include_files( sys/types.h   ECCODES_HAVE_SYS_TYPES_H )
+check_include_files( sys/stat.h    ECCODES_HAVE_SYS_STAT_H )
+check_include_files( fcntl.h       ECCODES_HAVE_FCNTL_H )
+check_include_files( unistd.h      ECCODES_HAVE_UNISTD_H )
+
+check_symbol_exists( fseeko          "stdio.h"    ECCODES_HAVE_FSEEKO )
+check_symbol_exists( posix_memalign  "stdlib.h"   ECCODES_HAVE_POSIX_MEMALIGN )
+check_symbol_exists( fmemopen        "stdio.h"    ECCODES_HAVE_FMEMOPEN )
+check_symbol_exists( funopen         "stdio.h"    ECCODES_HAVE_FUNOPEN )
+check_symbol_exists( realpath        "stdlib.h"   ECCODES_HAVE_REALPATH )
+check_symbol_exists( fsync           "unistd.h"   ECCODES_HAVE_FSYNC)
+check_symbol_exists( fdatasync       "unistd.h"   ECCODES_HAVE_FDATASYNC)
 
 check_c_source_compiles(
       " typedef int foo_t;
@@ -113,9 +115,15 @@ ecbuild_add_option( FEATURE NETCDF
     NO_TPL )
 
 find_package( AEC )
+if(NOT DEFINED ENABLE_AEC AND NOT AEC_FOUND)
+  ecbuild_critical("AEC library was not found.\n"
+                   "AEC (Adaptive Entropy Coding) provides the WMO GRIB CCSDS compression and decompression of data. "
+                   "This is highly recommended from ecCodes >= 2.25.0"
+                   "\nTo force the build without it, use -DENABLE_AEC=OFF")
+endif()
 ecbuild_add_option( FEATURE AEC
     DESCRIPTION "Support for Adaptive Entropy Coding"
-    DEFAULT OFF
+    DEFAULT ON
     CONDITION AEC_FOUND )
 
 ecbuild_find_python( VERSION 2.6 NO_LIBS )
@@ -183,7 +191,7 @@ ecbuild_add_option( FEATURE MEMORY_MANAGEMENT   DESCRIPTION "Enable memory manag
 ecbuild_add_option( FEATURE ALIGN_MEMORY        DESCRIPTION "Enable memory alignment"  DEFAULT OFF ADVANCED )
 ecbuild_add_option( FEATURE TIMER               DESCRIPTION "Enable timer" DEFAULT OFF ADVANCED )
 ecbuild_add_option( FEATURE ECCODES_THREADS     DESCRIPTION "Enable POSIX threads" DEFAULT OFF ADVANCED )
-ecbuild_add_option( FEATURE ECCODES_OMP_THREADS DESCRIPTION "Enable OMP threads" DEFAULT OFF ADVANCED )
+ecbuild_add_option( FEATURE ECCODES_OMP_THREADS DESCRIPTION "Enable OpenMP threads" DEFAULT OFF ADVANCED )
 ecbuild_add_option( FEATURE EXTRA_TESTS         DESCRIPTION "Enable extended regression testing" DEFAULT OFF ADVANCED )
 
 ###############################################################################
@@ -285,11 +293,11 @@ endif()
 
 # Cannot have both options
 if( HAVE_ECCODES_THREADS AND HAVE_ECCODES_OMP_THREADS )
-    ecbuild_critical("Cannot enable both POSIX threads and OMP! Please specify just one option")
+    ecbuild_critical("Cannot enable both POSIX threads and OpenMP! Please specify just one option")
 endif()
 
-ecbuild_info(" GRIB_PTHREADS=${GRIB_PTHREADS}")
-ecbuild_info(" GRIB_OMP_THREADS=${GRIB_OMP_THREADS}")
+ecbuild_info(" POSIX THREADS = ${GRIB_PTHREADS}")
+ecbuild_info(" OpenMP THREADS= ${GRIB_OMP_THREADS}")
 
 set( GRIB_MEM_ALIGN 0 )
 if( ENABLE_ALIGN_MEMORY )
@@ -306,6 +314,11 @@ if( IEEE_LE )
     set( IEEE_BE 0 )
 endif()
 
+set( ECCODES_ON_LINUX_32BIT 0 )
+if( EC_OS_NAME MATCHES "linux" AND EC_OS_BITS EQUAL "32" )
+    set( ECCODES_ON_LINUX_32BIT 1 )
+endif()
+
 set( ECCODES_ON_WINDOWS 0 )
 if( EC_OS_NAME MATCHES "windows" )
     # Symbols need to be explicitly exported on Windows so we can link to dlls.

NOTICE

@@ -3,7 +3,7 @@ ECMWF ecCodes
 
 This product includes software developed at ECMWF (http://www.ecmwf.int).
 
-Parts of the definitions provided by WMO (http://www.wmo.int/pages/index_en.html)
+Parts of the definitions provided by WMO (https://public.wmo.int/en)
 
 Microsoft Visual Studio support provided with the help of
 Finnish Meteorological Institute (FMI).

README.md

@@ -22,12 +22,12 @@ It is designed to provide the user with a simple set of functions to access data
 several formats with a key/value approach.
 
 Documentation can be found here:
-   https://software.ecmwf.int/wiki/display/ECC/ecCodes+Home
+   https://confluence.ecmwf.int/display/ECC/ecCodes+Home
 
 INSTALLATION
 ------------
 
-1. Download ecCodes from https://software.ecmwf.int/wiki/display/ECC/Releases
+1. Download ecCodes from https://confluence.ecmwf.int/display/ECC/Releases
 
 2. Unpack distribution:
    ```
@@ -65,7 +65,7 @@ To add the Python3 bindings, use pip3 install from PyPI as follows:
 
 
 For more details, please see:
-https://software.ecmwf.int/wiki/display/ECC/ecCodes+installation
+https://confluence.ecmwf.int/display/ECC/ecCodes+installation
 
 If you encounter any problems please send an e-mail with your problem to:
 

bamboo/CLANG-env.sh

@@ -16,5 +16,6 @@ module unload libemos
 module unload metview
 
 module load cmake/3.16.5
+module load aec
 
 module switch gnu clang

bamboo/GCC-env.sh

@@ -15,3 +15,5 @@ module unload libemos
 module unload metview
 
 module load cmake/3.16.5
+module load aec
+

bamboo/INTEL-env.sh

@@ -16,5 +16,6 @@ module unload netcdf4
 module list
 
 module load cmake/3.16.5
+module load aec
 
 module switch gnu intel/17.0.3

cmake/FindAEC.cmake

@@ -18,14 +18,18 @@
 #
 #  AEC_DIR          - prefix path of the AEC installation
 #  AEC_PATH         - prefix path of the AEC installation
+#  LIBAEC_DIR
+#  libaec_DIR
+#  LIBAEC_PATH
+#  libaec_PATH
 
-find_path( AEC_INCLUDE_DIR szlib.h
-           PATHS ${AEC_DIR} ${AEC_PATH} ENV AEC_DIR ENV AEC_PATH
+find_path( AEC_INCLUDE_DIR libaec.h
+           PATHS ${AEC_DIR} ${AEC_PATH} ${LIBAEC_DIR} ${libaec_DIR} ${LIBAEC_PATH} ${libaec_PATH} ENV AEC_DIR ENV AEC_PATH ENV LIBAEC_DIR ENV libaec_DIR ENV LIBAEC_PATH ENV libaec_PATH
            PATH_SUFFIXES include include/aec NO_DEFAULT_PATH )
-find_path( AEC_INCLUDE_DIR szlib.h PATH_SUFFIXES include include/aec )
+find_path( AEC_INCLUDE_DIR libaec.h PATH_SUFFIXES include include/aec )
 
 find_library( AEC_LIBRARY  NAMES aec
-              PATHS ${AEC_DIR} ${AEC_PATH} ENV AEC_DIR ENV AEC_PATH
+              PATHS ${AEC_DIR} ${AEC_PATH} ${LIBAEC_DIR} ${libaec_DIR} ${LIBAEC_PATH} ${libaec_PATH} ENV AEC_DIR ENV AEC_PATH ENV LIBAEC_DIR ENV libaec_DIR ENV LIBAEC_PATH ENV libaec_PATH
               PATH_SUFFIXES lib lib64 lib/aec lib64/aec NO_DEFAULT_PATH )
 find_library( AEC_LIBRARY NAMES aec PATH_SUFFIXES lib lib64 lib/aec lib64/aec )
 

data/CMakeLists.txt

@@ -9,9 +9,11 @@ add_subdirectory(gts)
 
 file(READ "grib_data_files.txt" files_to_download)
 string(REGEX REPLACE "\n" ";" files_to_download "${files_to_download}")
-if( HAVE_AEC )
-    list(APPEND files_to_download ccsds.grib2)
-endif()
+
+# Download the CCSDS file even if HAVE_AEC feature is disabled.
+# This is for testing (See ECC-1352)
+list(APPEND files_to_download ccsds.grib2)
+
 
 if( ENABLE_EXTRA_TESTS )
     # Download all data files
@@ -35,7 +37,6 @@ list(APPEND other_files
     multi_step.txt
     no_bitmap.diff
     read_any.ok
-    scan_x_regular_gg_5_7_good.dump
     scan_x_regular_ll_5_4_good.dump
     scan_x_regular_ll_5_7_good.dump
     scan_x_regular_ll_8_4_good.dump

data/bitmap.diff

@@ -5,12 +5,12 @@ GRIB {
   # European Centre for Medium-Range Weather Forecasts (common/c-1.table)  
   centre = 98;
   generatingProcessIdentifier = 130;
-  # Temperature  (K)  (grib1/2.98.128.table)  
+  # Temperature (K)  (grib1/2.98.128.table)  
   indicatorOfParameter = 130;
-  # Hybrid level level number  (2 octets)  (grib1/local/ecmf/3.table , grib1/3.table)  
+  # Hybrid level level number (2 octets)  (grib1/local/ecmf/3.table , grib1/3.table)  
   indicatorOfTypeOfLevel = 109;
   level = 1;
-  # Forecast product valid at reference time + P1  (P1>0)  (grib1/local/ecmf/5.table , grib1/5.table)  
+  # Forecast product valid at reference time + P1 (P1>0)  (grib1/local/ecmf/5.table , grib1/5.table)  
   timeRangeIndicator = 0;
   # Unknown code table entry (grib1/0.ecmf.table)  
   subCentre = 0;

data/no_bitmap.diff

@@ -5,12 +5,12 @@ GRIB {
   # European Centre for Medium-Range Weather Forecasts (common/c-1.table)  
   centre = 98;
   generatingProcessIdentifier = 130;
-  # Temperature  (K)  (grib1/2.98.128.table)  
+  # Temperature (K)  (grib1/2.98.128.table)  
   indicatorOfParameter = 130;
-  # Hybrid level level number  (2 octets)  (grib1/local/ecmf/3.table , grib1/3.table)  
+  # Hybrid level level number (2 octets)  (grib1/local/ecmf/3.table , grib1/3.table)  
   indicatorOfTypeOfLevel = 109;
   level = 1;
-  # Forecast product valid at reference time + P1  (P1>0)  (grib1/local/ecmf/5.table , grib1/5.table)  
+  # Forecast product valid at reference time + P1 (P1>0)  (grib1/local/ecmf/5.table , grib1/5.table)  
   timeRangeIndicator = 0;
   # Unknown code table entry (grib1/0.ecmf.table)  
   subCentre = 0;

data/tigge_pf_ecmwf.grib2.ref

@@ -1,16 +1,16 @@
 ecmf 165 10u m s**-1 10 metre U wind component
 ecmf 166 10v m s**-1 10 metre V wind component
 ecmf 59 cape J kg**-1 Convective available potential energy
-ecmf 156 gh gpm Geopotential Height
+ecmf 156 gh gpm Geopotential height
 ecmf 151 msl Pa Mean sea level pressure
 ecmf 179 ttr J m**-2 Top net thermal radiation
 ecmf 3 pt K Potential temperature
 ecmf 60 pv K m**2 kg**-1 s**-1 Potential vorticity
 ecmf 235 skt K Skin temperature
 ecmf 228141 sd kg m**-2 Snow depth water equivalent
 ecmf 228144 sf kg m**-2 Snow Fall water equivalent
-ecmf 228039 sm kg m**-3 Soil Moisture
-ecmf 228139 st K Soil Temperature
+ecmf 228039 sm kg m**-3 Soil moisture
+ecmf 228139 st K Soil temperature
 ecmf 133 q kg kg**-1 Specific humidity
 ecmf 189 sund s Sunshine duration
 ecmf 168 2d K 2 metre dewpoint temperature

definitions/add_params_from_tsv.pl

@@ -25,7 +25,7 @@
 use Getopt::Long;
 
 my $SANITY_CHECK     = 0;
-my $WRITE_TO_FILES   = 1;
+my $WRITE_TO_FILES   = 0;
 my $WRITE_TO_PARAMDB = 0; # Be careful. Fill in $contactId before proceeding
 
 # Process arguments. Must be at least one file
@@ -113,7 +113,7 @@
             next;
         }
         $lcount++;
-        ($paramId, $shortName) = split(/\t/);
+        ($paramId, $shortName, $name, $units) = split(/\t/);
 
         die "Error: shortName=$shortName is duplicated (line ", $lcount+1, ")\n" if (exists $map_sn{$shortName});
         $map_sn{$shortName}++; # increment count in shortName map
@@ -125,6 +125,10 @@
 
         my $x = $dbh->selectrow_array("select * from param.param where id = ?",undef,$paramId);
         die "Error: paramId=$x exists in the database (line ", $lcount+1, ")\n" if (defined $x);
+
+        # Will die if it fails
+        get_db_units_code($units);
+
         $x = $dbh->selectrow_array("select shortName from param.param where shortName = ?",undef,$shortName);
         die "Error: shortName=$x exists in the database (line ", $lcount+1, ")\n" if (defined $x);
     }
@@ -174,6 +178,11 @@
     $scaleFactorWL2 = undef if ($scaleFactorWL2 =~ /missing/);
     $scaledValueWL2 = undef if ($scaledValueWL2 =~ /missing/);
 
+    $scaledValue1 = undef if ($scaledValue1 =~ /missing/);
+    $scaleFactor1 = undef if ($scaleFactor1 =~ /missing/);
+    $scaledValue2 = undef if ($scaledValue2 =~ /missing/);
+    $scaleFactor2 = undef if ($scaleFactor2 =~ /missing/);
+
     if ($WRITE_TO_FILES) {
         write_out_file(\*OUT_PARAMID,   $name, $paramId);
         write_out_file(\*OUT_SHORTNAME, $name, $shortName);
@@ -206,7 +215,7 @@
 
         die "Error: Both aerosolType and constituentType cannot be set!" if ($constit ne "" && $aero ne "");
         die "Error: No contact ID provided\n" if (!$contactId);
-        print "Inserting paramId $paramId (centre=$centre) ...\n";
+        print "Inserting paramId $paramId (centre=" . centre_as_str($centre) . ") ...\n";
         $dbh->do("insert into param(id,shortName,name,units_id,insert_date,update_date,contact) values (?,?,?,?,?,?,?)",undef,
             $paramId, $shortName, $name , $units_code, $today_date, $today_date, $contactId) or die $dbh->errstr;
 
@@ -220,10 +229,20 @@
         $dbh->do("insert into grib values (?,?,?,?,?,?)",undef, $paramId,$edition,$centre,5, $pnumber,0);
         $dbh->do("insert into grib values (?,?,?,?,?,?)",undef, $paramId,$edition,$centre,6, $type1,0)        if ($type1 ne "");
         $dbh->do("insert into grib values (?,?,?,?,?,?)",undef, $paramId,$edition,$centre,13,$type2,0)        if ($type2 ne "");
-        $dbh->do("insert into grib values (?,?,?,?,?,?)",undef, $paramId,$edition,$centre,9, $scaledValue1,0) if ($scaledValue1 ne "");
-        $dbh->do("insert into grib values (?,?,?,?,?,?)",undef, $paramId,$edition,$centre,7, $scaleFactor1,0) if ($scaleFactor1 ne "");
-        $dbh->do("insert into grib values (?,?,?,?,?,?)",undef, $paramId,$edition,$centre,14,$scaledValue2,0) if ($scaledValue2 ne "");
-        $dbh->do("insert into grib values (?,?,?,?,?,?)",undef, $paramId,$edition,$centre,15,$scaleFactor2,0) if ($scaleFactor2 ne "");
+
+        # Either missing or has a value
+        if (! defined $scaledValue1 || $scaledValue1 ne "") {
+            $dbh->do("insert into grib values (?,?,?,?,?,?)",undef, $paramId,$edition,$centre,9, $scaledValue1,0);
+        }
+        if (! defined $scaleFactor1 || $scaleFactor1 ne "") {
+            $dbh->do("insert into grib values (?,?,?,?,?,?)",undef, $paramId,$edition,$centre,7, $scaleFactor1,0);
+        }
+        if (! defined $scaledValue2 || $scaledValue2 ne "") {
+            $dbh->do("insert into grib values (?,?,?,?,?,?)",undef, $paramId,$edition,$centre,14,$scaledValue2,0);
+        }
+        if (! defined $scaleFactor2 || $scaleFactor2 ne "") {
+            $dbh->do("insert into grib values (?,?,?,?,?,?)",undef, $paramId,$edition,$centre,15,$scaleFactor2,0);
+        }
         $dbh->do("insert into grib values (?,?,?,?,?,?)",undef, $paramId,$edition,$centre,11,$stat,0)         if ($stat ne "");
 
         $dbh->do("insert into grib values (?,?,?,?,?,?)",undef, $paramId,$edition,$centre,46,$aero,0)         if ($aero ne "");
@@ -268,6 +287,12 @@
 }
 
 # -------------------------------------------------------------------
+sub centre_as_str {
+    my $cc = shift;
+    return "WMO"   if ($cc eq $centre_wmo);
+    return "ECMWF" if ($cc eq $centre_ecmwf);
+    return "Unknown";
+}
 sub get_db_units_code {
     my $u = shift;
     my $unit_id = $dbh->selectrow_array("select id from units where name = ?",undef,$u);
@@ -290,6 +315,12 @@ sub write_out_file {
     # Optional keys
     print $outfile "  typeOfFirstFixedSurface = $type1 ;\n"                if ($type1 ne "");
     print $outfile "  typeOfSecondFixedSurface = $type2 ;\n"               if ($type2 ne "");
+
+    $scaledValue1 = "missing()" if (! defined $scaledValue1);
+    $scaledValue2 = "missing()" if (! defined $scaledValue2);
+    $scaleFactor1 = "missing()" if (! defined $scaleFactor1);
+    $scaleFactor2 = "missing()" if (! defined $scaleFactor2);
+
     print $outfile "  scaledValueOfFirstFixedSurface = $scaledValue1 ;\n"  if ($scaledValue1 ne "");
     print $outfile "  scaleFactorOfFirstFixedSurface = $scaleFactor1 ;\n"  if ($scaleFactor1 ne "");
     print $outfile "  scaledValueOfSecondFixedSurface = $scaledValue2 ;\n" if ($scaledValue2 ne "");
@@ -357,6 +388,8 @@ sub usage {
    print <<USAGE;
 
 Usage: $0 [-s] [-f] [-p] file.tsv
+       Input has to be a tab-separated values (TSV) file
+
        -s  Perform sanity checks and exit
        -f  Write out def files (paramId.def, name.def etc)
        -p  Write to Parameter Database (Be careful!)