Feature/si 2694 add validation to definitionsyaml in model garage

.github/workflows/generate.yml

@@ -0,0 +1,39 @@
+name: generate files
+on:
+  pull_request:
+    branches: [ '**' ]
+env:
+  GOPRIVATE: github.com/DIMO-Network
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+jobs:
+  golangci:
+    runs-on: ubuntu-latest
+
+    name: lint
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: 1.22
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: go mod tidy
+        run: go mod tidy
+
+      - name: go mod verify
+        run: go mod verify
+
+      - name: generate
+        run: make generate
+
+      - name: porcelain
+        shell: bash
+        run: |
+          dirty_files="$(git status --porcelain)"
+          if [[ `git status --porcelain` ]]; then
+            echo "The following files are dirty after running generators:"
+            echo "${dirty_files}"
+            exit 1
+          fi

.golangci.yml

@@ -56,3 +56,10 @@ issues:
     - path: convert-funcs_test\.go
       linters:
         - dupl
+    # Exclude some linters from running on tests files.
+    - path: _test\.go
+      linters:
+        - funlen
+        - errcheck
+        - dupl
+        - gosec

README.md

@@ -40,36 +40,27 @@ The Model generation is handled by packages in `internal/codegen`. They are resp
 
 ```yaml
 # vspecName: The name of the VSpec field in the VSS schema
+# required
 - vspecName: DIMO.DefinitionID
-
-  # isArray: Whether the field is an array or not
-  # if null then the value is inferred from the vspec definition
-  isArray: null
-
-  # clickHouseType: The data type to use for ClickHouse Database.
-  #if empty then the type is inferred from the vspec definition
-  clickHouseType: ""
-
   # goType: The data type to use for Golang struct.
+  # available types: [float64, string]
   # if empty then the type is inferred from the vspec definition
   goType: ""
 
-  # gqlType: The data type to use for GraphQL schema.
-  # if empty then the type is inferred from the vspec definition
-  gqlType: ""
-
   # conversion: The mapping of the original data to the VSpec field
   conversion:
     # originalName: The name of the field in the original data
+    # required
     originalName: data.definitionID
 
     # originalType: The data type of the field in the original data
     originalType: string
 
-    # isArray: Whether the field is an array or not
+    # isArray: Whether the original field is an array or not
     isArray: false
 
   # requiredPrivileges: The list of privileges required to access the field
+  # required
   requiredPrivileges:
     - VEHICLE_NON_LOCATION_DATA
 ```

go.mod

@@ -42,7 +42,6 @@ require (
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/klauspost/compress v1.17.7 // indirect
 	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
-	github.com/madflojo/testcerts v1.1.1 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mfridman/interpolate v0.0.2 // indirect
 	github.com/moby/docker-image-spec v1.3.1 // indirect

go.sum

@@ -81,8 +81,6 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
 github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
-github.com/madflojo/testcerts v1.1.1 h1:YsSHWV79nMNZK0mJtwXjKoYHjJEbLPFefR8TxmmWupY=
-github.com/madflojo/testcerts v1.1.1/go.mod h1:MW8sh39gLnkKh4K0Nc55AyHEDl9l/FBLDUsQhpmkuo0=
 github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
 github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=

pkg/schema/load.go

@@ -37,15 +37,18 @@ func LoadSignalsCSV(r io.Reader) ([]*SignalInfo, error) {
 // LoadDefinitionFile loads the definitions from a definitions.yaml file.
 func LoadDefinitionFile(r io.Reader) (*Definitions, error) {
 	decoder := yaml.NewDecoder(r)
-	var transInfos []*DefinitionInfo
-	err := decoder.Decode(&transInfos)
+	var defInfos []*DefinitionInfo
+	err := decoder.Decode(&defInfos)
 	if err != nil {
 		return nil, fmt.Errorf("failed to decode json: %w", err)
 	}
 	definitions := &Definitions{
 		FromName: map[string]*DefinitionInfo{},
 	}
-	for _, info := range transInfos {
+	for _, info := range defInfos {
+		if err := Validate(info); err != nil {
+			return nil, fmt.Errorf("error validating definitions: %w", err)
+		}
 		definitions.FromName[info.VspecName] = info
 	}
 

pkg/schema/signal.go

@@ -55,8 +55,6 @@ type ConversionInfo struct {
 // DefinitionInfo contains the definition information for a field.
 type DefinitionInfo struct {
 	VspecName          string            `json:"vspecName"          yaml:"vspecName"`
-	IsArray            *bool             `json:"isArray"            yaml:"isArray"`
-	ClickHouseType     string            `json:"clickHouseType"     yaml:"clickHouseType"`
 	GoType             string            `json:"goType"             yaml:"goType"`
 	Conversions        []*ConversionInfo `json:"conversions"        yaml:"conversions"`
 	RequiredPrivileges []string          `json:"requiredPrivileges" yaml:"requiredPrivileges"`
@@ -117,8 +115,8 @@ func NewSignalInfo(record []string) *SignalInfo {
 
 // MergeWithDefinition merges the signal with the definition information.
 func (s *SignalInfo) MergeWithDefinition(definition *DefinitionInfo) {
-	if definition.IsArray != nil {
-		s.IsArray = *definition.IsArray
+	if definition.GoType != "" {
+		s.BaseGoType = definition.GoType
 	}
 	if len(definition.Conversions) != 0 {
 		s.Conversions = definition.Conversions
@@ -133,9 +131,6 @@ func (s *SignalInfo) MergeWithDefinition(definition *DefinitionInfo) {
 
 // GOType returns the golang type of the signal.
 func (s *SignalInfo) GOType() string {
-	if s.IsArray {
-		return "[]" + s.BaseGoType
-	}
 	return s.BaseGoType
 }