feat: read template 8 (#8)

.gitignore

@@ -1,2 +1,3 @@
 /venv/
 .vscode/
+pkg/testdata/**/gfs.*

go.mod

@@ -5,6 +5,7 @@ go 1.22.2
 require (
 	github.com/icza/bitio v1.1.0
 	github.com/stretchr/testify v1.9.0
+	golang.org/x/sync v0.8.0
 )
 
 require (

go.sum

@@ -8,6 +8,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
+golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

pkg/grib2/drt/grid_point/simple.go

@@ -40,7 +40,13 @@ func (sp *SimplePacking) ReadAllData(r datapacking.BitReader) ([]float64, error)
 		scaleFunc = sp.ScaleFunc()
 	)
 
-	for {
+	if sp.Bits == 0 {
+		for range sp.numVals {
+			values = append(values, scaleFunc(0))
+		}
+	}
+
+	for sp.Bits > 0 {
 		bitsVal, err := r.ReadBits(sp.Bits)
 		if errors.Is(err, io.EOF) {
 			break

pkg/grib2/grib2_test.go

@@ -1,6 +1,7 @@
 package grib2_test
 
 import (
+	"errors"
 	"io"
 	"os"
 	"testing"
@@ -15,6 +16,7 @@ import (
 	"github.com/scorix/grib-go/pkg/grib2/regulation"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"golang.org/x/sync/errgroup"
 )
 
 func assertSection(t testing.TB, sec grib2.Section, number int, length int) {
@@ -130,21 +132,21 @@ func TestGrib_ReadSection_SimplePacking(t *testing.T) {
 			test: func(t *testing.T, sec grib2.Section) {
 				assertSection(t, sec, 4, 34)
 				assertSection4(t, sec, &pdt.Template0{
-					ParameterCategory:               0,
-					ParameterNumber:                 0,
-					TypeOfGeneratingProcess:         2,
-					BackgroundProcess:               -1,
-					GeneratingProcessIdentifier:     -1,
-					HoursAfterDataCutoff:            -1,
-					MinutesAfterDataCutoff:          -1,
-					IndicatorOfUnitForForecastTime:  1,
-					ForecastTime:                    0,
-					TypeOfFirstFixedSurface:         1,
-					ScaleFactorOfFirstFixedSurface:  -1,
-					ScaledValueOfFirstFixedSurface:  -1,
-					TypeOfSecondFixedSurface:        -1,
-					ScaleFactorOfSecondFixedSurface: -1,
-					ScaledValueOfSecondFixedSurface: -1,
+					ParameterCategory:       0,
+					ParameterNumber:         0,
+					TypeOfGeneratingProcess: 2,
+					BackgroundProcess:       -1,
+					AnalysisOrForecastGeneratingProcessIdentified: -1,
+					HoursAfterDataCutoff:                          -1,
+					MinutesAfterDataCutoff:                        -1,
+					IndicatorOfUnitForForecastTime:                1,
+					ForecastTime:                                  0,
+					TypeOfFirstFixedSurface:                       1,
+					ScaleFactorOfFirstFixedSurface:                -1,
+					ScaledValueOfFirstFixedSurface:                -1,
+					TypeOfSecondFixedSurface:                      -1,
+					ScaleFactorOfSecondFixedSurface:               -1,
+					ScaledValueOfSecondFixedSurface:               -1,
 				})
 			},
 		},
@@ -256,21 +258,21 @@ func TestGrib_ReadSection_ComplexPacking(t *testing.T) {
 			test: func(t *testing.T, sec grib2.Section) {
 				assertSection(t, sec, 4, 34)
 				assertSection4(t, sec, &pdt.Template0{
-					ParameterCategory:               2,
-					ParameterNumber:                 2,
-					TypeOfGeneratingProcess:         2,
-					BackgroundProcess:               0,
-					GeneratingProcessIdentifier:     96,
-					HoursAfterDataCutoff:            0,
-					MinutesAfterDataCutoff:          0,
-					IndicatorOfUnitForForecastTime:  1,
-					ForecastTime:                    6,
-					TypeOfFirstFixedSurface:         103,
-					ScaleFactorOfFirstFixedSurface:  0,
-					ScaledValueOfFirstFixedSurface:  10,
-					TypeOfSecondFixedSurface:        -1,
-					ScaleFactorOfSecondFixedSurface: 0,
-					ScaledValueOfSecondFixedSurface: 0,
+					ParameterCategory:       2,
+					ParameterNumber:         2,
+					TypeOfGeneratingProcess: 2,
+					BackgroundProcess:       0,
+					AnalysisOrForecastGeneratingProcessIdentified: 96,
+					HoursAfterDataCutoff:                          0,
+					MinutesAfterDataCutoff:                        0,
+					IndicatorOfUnitForForecastTime:                1,
+					ForecastTime:                                  6,
+					TypeOfFirstFixedSurface:                       103,
+					ScaleFactorOfFirstFixedSurface:                0,
+					ScaledValueOfFirstFixedSurface:                10,
+					TypeOfSecondFixedSurface:                      -1,
+					ScaleFactorOfSecondFixedSurface:               0,
+					ScaledValueOfSecondFixedSurface:               0,
 				})
 			},
 		},
@@ -392,21 +394,21 @@ func TestGrib_ReadSection_ComplexPackingAndSpatialDifferencing(t *testing.T) {
 			test: func(t *testing.T, sec grib2.Section) {
 				assertSection(t, sec, 4, 34)
 				assertSection4(t, sec, &pdt.Template0{
-					ParameterCategory:               3,
-					ParameterNumber:                 196,
-					TypeOfGeneratingProcess:         2,
-					BackgroundProcess:               0,
-					GeneratingProcessIdentifier:     96,
-					HoursAfterDataCutoff:            0,
-					MinutesAfterDataCutoff:          0,
-					IndicatorOfUnitForForecastTime:  1,
-					ForecastTime:                    44,
-					TypeOfFirstFixedSurface:         1,
-					ScaleFactorOfFirstFixedSurface:  0,
-					ScaledValueOfFirstFixedSurface:  0,
-					TypeOfSecondFixedSurface:        -1,
-					ScaleFactorOfSecondFixedSurface: 0,
-					ScaledValueOfSecondFixedSurface: 0,
+					ParameterCategory:       3,
+					ParameterNumber:         196,
+					TypeOfGeneratingProcess: 2,
+					BackgroundProcess:       0,
+					AnalysisOrForecastGeneratingProcessIdentified: 96,
+					HoursAfterDataCutoff:                          0,
+					MinutesAfterDataCutoff:                        0,
+					IndicatorOfUnitForForecastTime:                1,
+					ForecastTime:                                  44,
+					TypeOfFirstFixedSurface:                       1,
+					ScaleFactorOfFirstFixedSurface:                0,
+					ScaledValueOfFirstFixedSurface:                0,
+					TypeOfSecondFixedSurface:                      -1,
+					ScaleFactorOfSecondFixedSurface:               0,
+					ScaledValueOfSecondFixedSurface:               0,
 				})
 			},
 		},
@@ -651,6 +653,7 @@ func TestGrib2_ReadMessage(t *testing.T) {
 		assert.Equal(t, 196, msg.GetParameterNumber())
 		assert.Equal(t, "2024-08-20T12:00:00Z", msg.GetTimestamp(time.UTC).Format(time.RFC3339))
 		assert.Equal(t, "2024-08-22T08:00:00Z", msg.GetForecastTime(time.UTC).Format(time.RFC3339))
+		assert.Equal(t, 0, msg.GetLevel())
 	}
 
 	{
@@ -659,3 +662,46 @@ func TestGrib2_ReadMessage(t *testing.T) {
 		assert.Nil(t, msg)
 	}
 }
+
+func TestGrib2_ReadMessages(t *testing.T) {
+	// aws s3 cp --no-sign-request s3://noaa-gfs-bdp-pds/gfs.20240820/12/atmos/gfs.t12z.pgrb2.0p25.f044 pkg/testdata/gfs.t12z.pgrb2.0p25.f044
+	const filename = "../testdata/gfs.t12z.pgrb2.0p25.f044"
+
+	s, err := os.Stat(filename)
+	if errors.Is(err, os.ErrNotExist) {
+		t.Skipf("%s not exist", filename)
+	}
+
+	t.Run(s.Name(), func(t *testing.T) {
+		f, err := os.Open(filename)
+		require.NoError(t, err)
+
+		g := grib.NewGrib2(f)
+		var msgs []grib2.Message
+
+		for i := 0; ; i++ {
+			msg, err := g.ReadMessage()
+			if errors.Is(err, io.EOF) {
+				break
+			}
+			require.NoError(t, err)
+			require.NotNil(t, msg)
+
+			msgs = append(msgs, msg)
+		}
+
+		var eg errgroup.Group
+		for i, msg := range msgs {
+			eg.Go(func() error {
+				data, err := msg.ReadData()
+				require.NoError(t, err)
+
+				t.Logf("count: %d, discipline: %d, category: %d, number: %d, forecast: %s, dataLen: %d", i+1, msg.GetDiscipline(), msg.GetParameterCategory(), msg.GetParameterNumber(), msg.GetForecastTime(time.UTC), len(data))
+
+				return nil
+			})
+		}
+
+		require.NoError(t, eg.Wait())
+	})
+}

pkg/grib2/message.go

@@ -1,13 +1,20 @@
 package grib2
 
-import "time"
+import (
+	"bytes"
+	"time"
+
+	"github.com/icza/bitio"
+)
 
 type Message interface {
 	GetDiscipline() int
 	GetParameterCategory() int
 	GetParameterNumber() int
 	GetTimestamp(loc *time.Location) time.Time
 	GetForecastTime(loc *time.Location) time.Time
+	GetLevel() int
+	ReadData() ([]float64, error)
 }
 
 type message struct {
@@ -41,3 +48,13 @@ func (m message) GetTimestamp(loc *time.Location) time.Time {
 func (m message) GetForecastTime(loc *time.Location) time.Time {
 	return m.GetTimestamp(loc).Add(m.sec4.GetForecastDuration())
 }
+
+func (m *message) GetLevel() int {
+	return m.sec4.GetLevel()
+}
+
+func (m *message) ReadData() ([]float64, error) {
+	tpl := m.sec5.GetDataRepresentationTemplate()
+
+	return tpl.ReadAllData(bitio.NewReader(bytes.NewReader(m.sec7.Data)))
+}

pkg/grib2/pdt/template.go

@@ -1,6 +1,7 @@
 package pdt
 
 import (
+	"bytes"
 	"encoding/binary"
 	"fmt"
 	"io"
@@ -11,23 +12,40 @@ type Template interface {
 	GetParameterCategory() int
 	GetParameterNumber() int
 	GetForecastDuration() time.Duration
+	GetLevel() int
 }
 
 type MissingTemplate struct{}
 
 func (m MissingTemplate) GetParameterCategory() int          { return -1 }
 func (m MissingTemplate) GetParameterNumber() int            { return -1 }
 func (m MissingTemplate) GetForecastDuration() time.Duration { return 0 }
+func (m MissingTemplate) GetLevel() int                      { return 0 }
 
 func ReadTemplate(r io.Reader, n uint16) (Template, error) {
 	switch n {
 	case 0:
-		var tpl template0
-		if err := binary.Read(r, binary.BigEndian, &tpl); err != nil {
+		t0, err := readTemplate0(r)
+		if err != nil {
 			return nil, err
 		}
 
-		return tpl.Export(), nil
+		return t0.Export(), nil
+
+	case 8:
+		t0, err := readTemplate0(r)
+		if err != nil {
+			return nil, err
+		}
+
+		t8, err := readTemplate8(r, t0)
+		if err != nil {
+			return nil, err
+		}
+
+		t8.template0 = t0
+
+		return t8.Export(), nil
 
 	case 255:
 		return &MissingTemplate{}, nil
@@ -36,3 +54,28 @@ func ReadTemplate(r io.Reader, n uint16) (Template, error) {
 		return nil, fmt.Errorf("unsupported product definition template: %d", n)
 	}
 }
+
+func readTemplate0(r io.Reader) (*template0, error) {
+	var tpl template0
+	if err := binary.Read(r, binary.BigEndian, &tpl); err != nil {
+		return nil, err
+	}
+
+	return &tpl, nil
+}
+
+func readTemplate8(r io.Reader, t0 *template0) (*template8, error) {
+	var tpl template8
+	if err := binary.Read(r, binary.BigEndian, &tpl.template8fields); err != nil {
+		return nil, err
+	}
+
+	tpl.template0 = t0
+
+	bs := tpl.template8fields.GetAdditionalTimeRangeSpecifications()
+	if _, err := io.CopyN(bytes.NewBuffer(bs), r, int64(len(bs))); err != nil {
+		return nil, err
+	}
+
+	return &tpl, nil
+}