Add multidocument support to specs

Signed-off-by: Igor Shishkin <me@teran.dev>

ceph/config/spec/spec.go

@@ -2,50 +2,51 @@ package spec
 
 import (
 	"encoding/json"
+	"io"
 	"os"
-	"path/filepath"
-	"strings"
 
 	"github.com/pkg/errors"
 	yaml "gopkg.in/yaml.v3"
 )
 
-type description struct {
+type Description struct {
 	Kind string          `json:"kind"`
 	Spec json.RawMessage `json:"spec"`
 }
 
-func NewFromDescription(filename string) (string, json.RawMessage, error) {
-	desc := description{}
+type yamlIntermediate struct {
+	Kind string `yaml:"kind"`
+	Spec any    `yaml:"spec"`
+}
 
-	data, err := os.ReadFile(filename)
+func NewFromDescription(filename string) ([]Description, error) {
+	fp, err := os.Open(filename)
 	if err != nil {
-		return "", nil, errors.Wrap(err, "error reading configuration file")
+		return nil, errors.Wrap(err, "error opening spec file")
 	}
+	defer fp.Close()
 
-	switch strings.ToLower(filepath.Ext(filename)) {
-	case ".yml", ".yaml":
-		type intermediate struct {
-			Kind string `yaml:"kind"`
-			Spec any    `yaml:"spec"`
-		}
-
-		d := intermediate{}
-		err := yaml.Unmarshal(data, &d)
+	docs := []Description{}
+	dec := yaml.NewDecoder(fp)
+	for {
+		v := yamlIntermediate{}
+		err := dec.Decode(&v)
 		if err != nil {
-			return "", nil, errors.Wrap(err, "error unmarshaling intermediate configuration")
+			if errors.Is(err, io.EOF) {
+				break
+			}
+			return nil, errors.Wrap(err, "error unmarshaling document")
 		}
-
-		spec, err := json.Marshal(d.Spec)
+		spec, err := json.Marshal(v.Spec)
 		if err != nil {
-			return "", nil, errors.Wrap(err, "error marshaling intermediate configuration")
+			return nil, errors.Wrap(err, "error marshaling intermediate data structure")
 		}
-		return d.Kind, json.RawMessage(spec), nil
-	case ".json":
-		// skip since supported natively
-	default:
-		return "", nil, errors.Errorf("unexpected file format: `%s`", filepath.Ext(filename))
+
+		docs = append(docs, Description{
+			Kind: v.Kind,
+			Spec: json.RawMessage(spec),
+		})
 	}
 
-	return desc.Kind, desc.Spec, json.Unmarshal(data, &desc)
+	return docs, nil
 }

ceph/config/spec/spec_test.go

@@ -6,20 +6,26 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func TestNewFromDescriptionYAML(t *testing.T) {
+func TestNewFromDescriptionSingle(t *testing.T) {
 	r := require.New(t)
 
-	kind, spec, err := NewFromDescription("testdata/sample_NewFromDescriptionYAML.yaml")
+	descs, err := NewFromDescription("testdata/sample_NewFromDescriptionSingle.yaml")
 	r.NoError(err)
-	r.Equal("CephConfig", kind)
-	r.JSONEq(`{"global":{"rbd_cache":"true"},"osd":{"rocksdb_perf":"true"}}`, string(spec))
+	r.Len(descs, 1)
+	r.Equal("CephConfig", descs[0].Kind)
+	r.JSONEq(`{"global":{"rbd_cache":"true"},"osd":{"rocksdb_perf":"true"}}`, string(descs[0].Spec))
 }
 
-func TestNewFromDescriptionJSON(t *testing.T) {
+func TestNewFromDescriptionMulti(t *testing.T) {
 	r := require.New(t)
 
-	kind, spec, err := NewFromDescription("testdata/sample_NewFromDescriptionJSON.json")
+	descs, err := NewFromDescription("testdata/sample_NewFromDescriptionMulti.yaml")
 	r.NoError(err)
-	r.Equal("CephConfig", kind)
-	r.JSONEq(`{"global":{"rbd_cache":"true"},"osd":{"rocksdb_perf":"true"}}`, string(spec))
+	r.Len(descs, 2)
+
+	r.Equal("CephConfig", descs[0].Kind)
+	r.JSONEq(`{"global":{"rbd_cache":"true"},"osd":{"rocksdb_perf":"true"}}`, string(descs[0].Spec))
+
+	r.Equal("CephOSDConfig", descs[1].Kind)
+	r.JSONEq(`{"allow_crimson":true}`, string(descs[1].Spec))
 }

ceph/config/spec/testdata/sample_NewFromDescriptionMulti.yaml

@@ -0,0 +1,11 @@
+---
+kind: CephConfig
+spec:
+  global:
+    rbd_cache: "true"
+  osd:
+    rocksdb_perf: "true"
+---
+kind: CephOSDConfig
+spec:
+  allow_crimson: true

commands/apply/apply.go

@@ -18,34 +18,36 @@ type ApplyConfig struct {
 }
 
 func Apply(ctx context.Context, ac ApplyConfig) error {
-	kind, specData, err := spec.NewFromDescription(ac.SpecFile)
+	descs, err := spec.NewFromDescription(ac.SpecFile)
 	if err != nil {
 		return err
 	}
 
-	switch strings.ToLower(kind) {
-	case "cephconfig":
-		cfg, err := cephconfig.New(specData)
-		if err != nil {
-			return err
+	for _, desc := range descs {
+		switch strings.ToLower(desc.Kind) {
+		case "cephconfig":
+			cfg, err := cephconfig.New(desc.Spec)
+			if err != nil {
+				return err
+			}
+
+			if err := ac.Service.ApplyCephConfig(ctx, cfg); err != nil {
+				return err
+			}
+
+		case "cephosdconfig":
+			cfg, err := cephosdconfig.New(desc.Spec)
+			if err != nil {
+				return err
+			}
+
+			if err := ac.Service.ApplyCephOSDConfig(ctx, cfg); err != nil {
+				return err
+			}
+
+		default:
+			return errors.Errorf("unexpected specification kind: `%s`", desc.Kind)
 		}
-
-		if err := ac.Service.ApplyCephConfig(ctx, cfg); err != nil {
-			return err
-		}
-
-	case "cephosdconfig":
-		cfg, err := cephosdconfig.New(specData)
-		if err != nil {
-			return err
-		}
-
-		if err := ac.Service.ApplyCephOSDConfig(ctx, cfg); err != nil {
-			return err
-		}
-
-	default:
-		return errors.Errorf("unexpected specification kind: `%s`", kind)
 	}
 
 	return nil

commands/diff/diff.go

@@ -22,59 +22,61 @@ type DiffConfig struct {
 }
 
 func Diff(ctx context.Context, ac DiffConfig) error {
-	kind, specData, err := spec.NewFromDescription(ac.SpecFile)
+	descs, err := spec.NewFromDescription(ac.SpecFile)
 	if err != nil {
 		return err
 	}
 
-	switch strings.ToLower(kind) {
-	case "cephconfig":
-		cfg, err := cephconfig.New(specData)
-		if err != nil {
-			return err
-		}
+	for _, desc := range descs {
+		switch strings.ToLower(desc.Kind) {
+		case "cephconfig":
+			cfg, err := cephconfig.New(desc.Spec)
+			if err != nil {
+				return err
+			}
 
-		changes, err := ac.Service.DiffCephConfig(ctx, cfg)
-		if err != nil {
-			return err
-		}
+			changes, err := ac.Service.DiffCephConfig(ctx, cfg)
+			if err != nil {
+				return err
+			}
 
-		for _, change := range changes {
-			log.WithFields(log.Fields{
-				"component": "command",
-			}).Tracef("change: %#v", change)
+			for _, change := range changes {
+				log.WithFields(log.Fields{
+					"component": "command",
+				}).Tracef("change: %#v", change)
 
-			switch change.Kind {
-			case models.CephConfigDifferenceKindAdd:
-				ac.Printer.Green("+ %s %s %s", change.Section, change.Key, *change.Value)
-			case models.CephConfigDifferenceKindChange:
-				ac.Printer.Yellow("~ %s %s %s -> %s", change.Section, change.Key, *change.OldValue, *change.Value)
-			case models.CephConfigDifferenceKindRemove:
-				ac.Printer.Red("- %s %s", change.Section, change.Key)
+				switch change.Kind {
+				case models.CephConfigDifferenceKindAdd:
+					ac.Printer.Green("+ %s %s %s", change.Section, change.Key, *change.Value)
+				case models.CephConfigDifferenceKindChange:
+					ac.Printer.Yellow("~ %s %s %s -> %s", change.Section, change.Key, *change.OldValue, *change.Value)
+				case models.CephConfigDifferenceKindRemove:
+					ac.Printer.Red("- %s %s", change.Section, change.Key)
+				}
 			}
-		}
 
-	case "cephosdconfig":
-		cfg, err := cephosdconfig.New(specData)
-		if err != nil {
-			return err
-		}
+		case "cephosdconfig":
+			cfg, err := cephosdconfig.New(desc.Spec)
+			if err != nil {
+				return err
+			}
 
-		changes, err := ac.Service.DiffCephOSDConfig(ctx, cfg)
-		if err != nil {
-			return err
-		}
+			changes, err := ac.Service.DiffCephOSDConfig(ctx, cfg)
+			if err != nil {
+				return err
+			}
 
-		for _, change := range changes {
-			log.WithFields(log.Fields{
-				"component": "command",
-			}).Tracef("change: %#v", change)
+			for _, change := range changes {
+				log.WithFields(log.Fields{
+					"component": "command",
+				}).Tracef("change: %#v", change)
 
-			ac.Printer.Yellow("~ %s %s -> %s", change.Key, change.OldValue, change.Value)
-		}
+				ac.Printer.Yellow("~ %s %s -> %s", change.Key, change.OldValue, change.Value)
+			}
 
-	default:
-		return errors.Errorf("unexpected specification kind: `%s`", kind)
+		default:
+			return errors.Errorf("unexpected specification kind: `%s`", desc.Kind)
+		}
 	}
 
 	return nil