Preserve YAML config file comments during a migration

Makefile

@@ -102,3 +102,18 @@ dst-db-psql: dst-db
 .PHONY: grep
 grep:
  grep -R --exclude-dir=.git --exclude-dir=dist ${ARGS} .
+
+.PHONY: manual-migration-test
+manual-migration-test: build
+ podman stop caweb1_0_0-pg16-dbms
+ podman stop caweb1_1_0-pg16-dbms
+ for dir in "$(SRC_DB_DIR)" "$(DST_DB_DIR)"; do \
+ podman unshare rm -rf "$$dir" && mkdir -p "$$dir"; \
+ done
+ $(MAKE) src-db dst-db
+ sleep 5
+ ./$(TARGET) db init-dev --config configs/sample-src-config.yaml
+ ./$(TARGET) db migrate configs/sample-src-config.yaml \
+ configs/sample-dst-config.yaml \
+ --config configs/sample-config.yaml
+ echo "Check configs/sample-config.yaml as the target config file."

configs/sample-config.yaml

@@ -2,19 +2,43 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at https://mozilla.org/MPL/2.0/.
-
 database:
+ # host IP address or name, although IP should be preferred as it
+ # requires no translation (even localhost from the /etc/hosts file)
  host: 127.0.0.1
+ # port number
  port: 5456
+ # database name which should contain complete semantic version
  name: caweb1_1_0
+ # passwords directory should contain a .pgpass or .pgpass.new file
+ # containing the PostgreSQL standard password lines following this
+ # format: 127.0.0.1:5456:caweb1_0_0:caweb:tHePaSsWoRd
  pass-dir: dist/.db/caweb1_1_0
  auth-method: scram-sha-256
 gin:
  logger: true
  recovery: true
+# The use cases specific configuration items are kept here which are
+# used for instantiation of those use cases. Although it works well
+# in this sample project, in a larger scale project, a different
+# set of categories which do not necessarily align with the use cases
+# may be useful. So, consider your project requirements and find
+# their natural configuration settings categories.
 usecases:
  cars:
  delay-of-old-parking-method: 15s
+# Versions are not settings themselves. For example, if we were talking
+# about the caweb Golang module version, it would find its place in a
+# const definition in some package (to be printed by a "version" command
+# as an example). However, following settings provide hints to know how
+# other settings are formatted. That is, a loader needs to check the
+# config version before knowing that "old-parking-method-delay" field
+# should be expected or "delay-of-old-parking-method" field. Similarly,
+# the database version informs us that which tables with which columns
+# are expected to be seen, so a proper migration plan may be set.
+# In this sense, these settings are immutable.
 versions:
+ # semantic version of the database schema
  database: 1.1.0
+ # semantic version of the configuration file itself
  config: 2.0.0

configs/sample-dst-config.yaml

@@ -5,17 +5,41 @@
 
 ---
 database:
+ # host IP address or name, although IP should be preferred as it
+ # requires no translation (even localhost from the /etc/hosts file)
  host: 127.0.0.1
+ # port number
  port: 5456
+ # database name which should contain complete semantic version
  name: caweb1_1_0
- role: caweb
+ # passwords directory should contain a .pgpass or .pgpass.new file
+ # containing the PostgreSQL standard password lines following this
+ # format: 127.0.0.1:5456:caweb1_0_0:caweb:tHePaSsWoRd
  pass-dir: dist/.db/caweb1_1_0
 gin:
  logger: true
  recovery: true
+# The use cases specific configuration items are kept here which are
+# used for instantiation of those use cases. Although it works well
+# in this sample project, in a larger scale project, a different
+# set of categories which do not necessarily align with the use cases
+# may be useful. So, consider your project requirements and find
+# their natural configuration settings categories.
 usecases:
  cars:
  delay-of-old-parking-method: 15s
+# Versions are not settings themselves. For example, if we were talking
+# about the caweb Golang module version, it would find its place in a
+# const definition in some package (to be printed by a "version" command
+# as an example). However, following settings provide hints to know how
+# other settings are formatted. That is, a loader needs to check the
+# config version before knowing that "old-parking-method-delay" field
+# should be expected or "delay-of-old-parking-method" field. Similarly,
+# the database version informs us that which tables with which columns
+# are expected to be seen, so a proper migration plan may be set.
+# In this sense, these settings are immutable.
 versions:
+ # semantic version of the database schema
  database: 1.1.0
+ # semantic version of the configuration file itself
  config: 2.0.0

configs/sample-src-config.yaml

@@ -4,12 +4,21 @@
 # file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
 ---
+# We can write comments in YAML format explaining settings
+# using pre-pended comment lines (comment lines which follow
+# a value and comments which are written at the end of a line
+# should be avoided as they cannot be preserved easily and unambiguously
+# during an automated migration operation).
+# Comments of the src config file will be ignored.
+# Comments of the dst config file will be used.
 database:
  host: 127.0.0.1
- port: 5455
+ port: 5455 # this comment which is written at the end of line is bad
+ # but this comment which is written before the name setting is good
  name: caweb1_0_0
- role: caweb
  pass-dir: dist/.db/caweb1_0_0
+ # and a comment following pass-dir is bad too as it may be thought
+ # to be written before the following gin setting!
 gin:
  logger: true
  recovery: true

pkg/adapter/config/cfg1/cfg1.go

@@ -22,6 +22,7 @@ import (
  "strings"
  "time"
 
+ "github.com/momeni/clean-arch/pkg/adapter/config/comment"
  "github.com/momeni/clean-arch/pkg/adapter/config/settings"
  "github.com/momeni/clean-arch/pkg/adapter/config/vers"
  "github.com/momeni/clean-arch/pkg/adapter/db/postgres"
@@ -65,6 +66,19 @@ type Config struct {
  // strings corresponding to this Config instance and its Database
  // target.
  Vers vers.Config `yaml:",inline"`
+
+ // Comments contains the YAML comment lines which are written right
+ // before the actual settings lines, aka head-comments.
+ // These comments are preserved for top-level settings and their
+ // children sequence and mapping YAML nodes. The Comments may be nil
+ // which will be ignored, or may be poppulated with some comments
+ // which will be preserved during a marshaling operation by the
+ // multi-database migration operation. Indeed, Comments field is
+ // only useful when the destination configuration file is loaded
+ // during a migration operation because the MergeConfig method
+ // preserves the destination Comments field, so the new comments
+ // may be seen in the target config file.
+ Comments *comment.Comment `yaml:"-"`
 }
 
 // Database contains the database related configuration settings.
@@ -517,13 +531,27 @@ func (c Cars) NewUseCase(
 // contents may change continually and their loading must be performed
 // by a separate method, such as LoadFromDB).
 func Load(data []byte) (*Config, error) {
- c := &Config{}
- if err := yaml.Unmarshal(data, c); err != nil {
+ n := &yaml.Node{}
+ if err := yaml.Unmarshal(data, n); err != nil {
  return nil, fmt.Errorf("unmarshalling yaml: %w", err)
  }
+ if l := len(n.Content); l != 1 {
+ return nil, fmt.Errorf(
+ "found %d children nodes, instead of 1 mapping child", l,
+ )
+ }
+ c := &Config{}
+ if err := n.Decode(c); err != nil {
+ return nil, fmt.Errorf("decoding yaml node: %w", err)
+ }
  if err := c.ValidateAndNormalize(); err != nil {
  return nil, fmt.Errorf("validating configs: %w", err)
  }
+ cmnts, err := comment.LoadFrom(n.Content[0])
+ if err != nil {
+ return nil, fmt.Errorf("parsing comments: %w", err)
+ }
+ c.Comments = cmnts
  return c, nil
 }
 
@@ -620,17 +648,27 @@ type Marshalled struct {
  Vers *vers.Marshalled `yaml:",inline"`
 }
 
-// MarshalYAML returns an instance of the Marshalled struct, as created
+// MarshalYAML computes an instance of the Marshalled struct, as created
 // by the Marshal method, so it may be marshalled instead of the `c`
 // Config instance. This replacement makes it possible to substitute
 // specific settings such as a slices of numbers in a vers.Config with
 // their alternative primitive data types and have control on the final
-// serialization result.
+// serialization result. Thereafter, it encodes *Marshalled as a yaml
+// node instance and saves the preserved head `c.Comments` (if any) into
+// the resulting *yaml.Node instance (and returns it as an interface{}).
 //
 // See the Marshal function for the reification details and how
 // marshaling logic can be distributed among nested Config structs.
 func (c *Config) MarshalYAML() (interface{}, error) {
- return c.Marshal(), nil
+ m := c.Marshal()
+ n := &yaml.Node{}
+ if err := n.Encode(m); err != nil {
+ return nil, fmt.Errorf("encoding *Marshalled as YAML: %w", err)
+ }
+ if err := c.Comments.SaveInto(n); err != nil {
+ return nil, fmt.Errorf("saving YAML nodes comments: %w", err)
+ }
+ return n, nil
 }
 
 // Marshal creates an instance of the Marshalled struct and fills it
@@ -718,6 +756,8 @@ func (c *Config) Clone() *Config {
 // unconditionally. The database version number will be set to its
 // latest supported version too, having the same major version as
 // specified in `c2` instance.
+// The Comments field takes its value from the `c2` instance, ignoring
+// comments of the `c` instance (if any).
 func (c *Config) MergeConfig(c2 *Config) error {
  c.Database = c2.Database
 
@@ -733,6 +773,7 @@ func (c *Config) MergeConfig(c2 *Config) error {
  return err
  }
  c.Vers.Versions.Database = sv
+ c.Comments = c2.Comments
  return nil
 }
 

pkg/adapter/config/cfg2/cfg2.go

@@ -17,6 +17,7 @@ import (
  "time"
 
  "github.com/momeni/clean-arch/pkg/adapter/config/cfg1"
+ "github.com/momeni/clean-arch/pkg/adapter/config/comment"
  "github.com/momeni/clean-arch/pkg/adapter/config/settings"
  "github.com/momeni/clean-arch/pkg/adapter/config/vers"
  "github.com/momeni/clean-arch/pkg/adapter/db/postgres/migration"
@@ -56,6 +57,19 @@ type Config struct {
  // strings corresponding to this Config instance and its Database
  // target.
  Vers vers.Config `yaml:",inline"`
+
+ // Comments contains the YAML comment lines which are written right
+ // before the actual settings lines, aka head-comments.
+ // These comments are preserved for top-level settings and their
+ // children sequence and mapping YAML nodes. The Comments may be nil
+ // which will be ignored, or may be poppulated with some comments
+ // which will be preserved during a marshaling operation by the
+ // multi-database migration operation. Indeed, Comments field is
+ // only useful when the destination configuration file is loaded
+ // during a migration operation because the MergeConfig method
+ // preserves the destination Comments field, so the new comments
+ // may be seen in the target config file.
+ Comments *comment.Comment `yaml:"-"`
 }
 
 // ConnectionPool creates a database connection pool using the
@@ -255,13 +269,27 @@ func (c Cars) NewUseCase(
 // contents may change continually and their loading must be performed
 // by a separate method, such as LoadFromDB).
 func Load(data []byte) (*Config, error) {
- c := &Config{}
- if err := yaml.Unmarshal(data, c); err != nil {
+ n := &yaml.Node{}
+ if err := yaml.Unmarshal(data, n); err != nil {
  return nil, fmt.Errorf("unmarshalling yaml: %w", err)
  }
+ if l := len(n.Content); l != 1 {
+ return nil, fmt.Errorf(
+ "found %d children nodes, instead of 1 mapping child", l,
+ )
+ }
+ c := &Config{}
+ if err := n.Decode(c); err != nil {
+ return nil, fmt.Errorf("decoding yaml node: %w", err)
+ }
  if err := c.ValidateAndNormalize(); err != nil {
  return nil, fmt.Errorf("validating configs: %w", err)
  }
+ cmnts, err := comment.LoadFrom(n.Content[0])
+ if err != nil {
+ return nil, fmt.Errorf("parsing comments: %w", err)
+ }
+ c.Comments = cmnts
  return c, nil
 }
 
@@ -358,17 +386,27 @@ type Marshalled struct {
  Vers *vers.Marshalled `yaml:",inline"`
 }
 
-// MarshalYAML returns an instance of the Marshalled struct, as created
+// MarshalYAML computes an instance of the Marshalled struct, as created
 // by the Marshal method, so it may be marshalled instead of the `c`
 // Config instance. This replacement makes it possible to substitute
 // specific settings such as a slices of numbers in a vers.Config with
 // their alternative primitive data types and have control on the final
-// serialization result.
+// serialization result. Thereafter, it encodes *Marshalled as a yaml
+// node instance and saves the preserved head `c.Comments` (if any) into
+// the resulting *yaml.Node instance (and returns it as an interface{}).
 //
 // See the Marshal function for the reification details and how
 // marshaling logic can be distributed among nested Config structs.
 func (c *Config) MarshalYAML() (interface{}, error) {
- return c.Marshal(), nil
+ m := c.Marshal()
+ n := &yaml.Node{}
+ if err := n.Encode(m); err != nil {
+ return nil, fmt.Errorf("encoding *Marshalled as YAML: %w", err)
+ }
+ if err := c.Comments.SaveInto(n); err != nil {
+ return nil, fmt.Errorf("saving YAML nodes comments: %w", err)
+ }
+ return n, nil
 }
 
 // Marshal creates an instance of the Marshalled struct and fills it
@@ -456,6 +494,8 @@ func (c *Config) Clone() *Config {
 // unconditionally. The database version number will be set to its
 // latest supported version too, having the same major version as
 // specified in `c2` instance.
+// The Comments field takes its value from the `c2` instance, ignoring
+// comments of the `c` instance (if any).
 func (c *Config) MergeConfig(c2 *Config) error {
  c.Database = c2.Database
  settings.OverwriteNil(&c.Gin.Logger, c2.Gin.Logger)
@@ -469,6 +509,7 @@ func (c *Config) MergeConfig(c2 *Config) error {
  return err
  }
  c.Vers.Versions.Database = sv
+ c.Comments = c2.Comments
  return nil
 }