feat: JSON schema compatibility check (#167)

* implement abstractted IsBackwardCompatible function

* update jsonschema dependency

* implement explore schema function

* error wrapper and helper functions

* bump go version to 1.20 to add support for generics

* refactored utils.go function name and few functions

* added static schema uri for equal comparison across schemas for location

* feat: implemented compatibility feature checks

* filter remote references from backward compatibility check

* feat: added check to restrict additional properties to to enforce open content model

* refactor compare schemas for better testability

* add compatibility test for additionalProperties check

* add test for compareSchema

* add property deleted test

* refactor: make type checks injectible for better testing

* added test for type check correctness

* refactor: move helper check functions to compatibility_helper file

* added test for reference check

* added tests for anyOf, allOf and oneOf conditionals

* added test to check field addition and added formatting

* adde checks for property addition

* added stub for tests

* feat: add schema for testing item schema

* implement item schema compatibility checks

* implement item schema exploration

* add nil check on type check executor

* fix: formatting and linting errors

* bump golang version for linting and test-server

* bump go version in release pipeline

* added new line to all json files

* fix: comments on compatibility files

* add parameter name for better readability

* added new line to json file

* improve correctness of backward compatibility by for oneOf and any Of conditions

* added formatting

* remove unused oneOf modified

* removed duplicate enum check

* fixed test when ref is absent

* fix: delete duplicated fields

.github/workflows/lint.yml

@@ -7,14 +7,14 @@ jobs:
     name: "Lint Go"
     runs-on: "ubuntu-latest"
     steps:
-      - uses: actions/setup-go@v2
+      - uses: actions/setup-go@v4
         with:
-          go-version: "1.17"
-      - uses: actions/checkout@v2
+          go-version: "1.20"
+      - uses: actions/checkout@v3
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v2
+        uses: golangci/golangci-lint-action@v3
         with:
-          skip-go-installation: true
+          version: v1.54
 
   codeql:
     name: "Analyze with CodeQL"

.github/workflows/release-server.yml

@@ -14,13 +14,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 0
       - name: Set up Go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v4
         with:
-          go-version: "1.16"
+          go-version: "1.20"
       - name: Login to DockerHub
         uses: docker/login-action@v1
         with:

.github/workflows/test-server.yaml

@@ -27,9 +27,9 @@ jobs:
           - 5432:5432
     steps:
       - name: Set up Go 1.x
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v4
         with:
-          go-version: ^1.16
+          go-version: ^1.20
         id: go
       - name: Install Protoc
         uses: arduino/setup-protoc@v1

formats/json/compatibility.go

@@ -0,0 +1,163 @@
+package json
+
+import (
+	"fmt"
+
+	"github.com/raystack/stencil/pkg/logger"
+	"github.com/santhosh-tekuri/jsonschema/v5"
+)
+
+const (
+	_ diffKind = iota
+	schemaDeleted
+	incompatibleTypes
+	requiredFieldChanged
+	propertyAddition
+	itemSchemaModification
+	itemSchemaAddition
+	itemsSchemaDeletion
+	subSchemaTypeModification
+	enumCreation
+	enumDeletion
+	enumElementDeletion
+	refChanged
+	anyOfModified
+	anyOfAdded
+	anyOfDeleted
+	anyOfElementAdded
+	anyOfElementDeleted
+	oneOfAdded
+	oneOfDeleted
+	oneOfElementAdded
+	oneOfElementDeleted
+	allOfModified
+	additionalPropertiesNotTrue
+)
+
+var backwardCompatibility = []diffKind{
+	schemaDeleted,
+	requiredFieldChanged,
+	itemSchemaAddition,
+	itemsSchemaDeletion,
+	incompatibleTypes,
+	itemSchemaModification,
+	subSchemaTypeModification,
+	enumCreation,
+	enumDeletion,
+	enumElementDeletion,
+	refChanged,
+	anyOfDeleted,
+	anyOfElementDeleted,
+	oneOfDeleted,
+	oneOfElementDeleted,
+	allOfModified,
+	additionalPropertiesNotTrue,
+}
+
+type SchemaCompareCheck func(prev, curr *jsonschema.Schema, err *compatibilityErr)
+type SchemaCheck func(curr *jsonschema.Schema, err *compatibilityErr)
+
+type TypeCheckSpec struct {
+	emptyTypeChecks  []SchemaCompareCheck
+	objectTypeChecks []SchemaCompareCheck
+	arrayTypeChecks  []SchemaCompareCheck
+}
+
+var (
+	emptyTypeChecks []SchemaCompareCheck = []SchemaCompareCheck{
+		checkAllOf, checkAnyOf, checkOneOf, checkEnum, checkRef,
+	}
+	objectTypeChecks []SchemaCompareCheck = []SchemaCompareCheck{
+		checkRequiredProperties, checkPropertyAddition,
+	}
+	/*
+		Array schemas can define subschemas for each index as well as for rest of the elements.
+		Hence, divided the two evaluation into two separate functions.
+	*/
+	arrayTypeChecks []SchemaCompareCheck = []SchemaCompareCheck{
+		checkItemSchema, checkRestOfItemsSchema,
+	}
+)
+
+var StandardTypeChecks TypeCheckSpec = TypeCheckSpec{emptyTypeChecks, objectTypeChecks, arrayTypeChecks}
+
+func compareSchemas(prevSchemaMap, currentSchemaMap map[string]*jsonschema.Schema, notAllowedChanges []diffKind,
+	schemaCompareFuncs []SchemaCompareCheck, schemaChecks []SchemaCheck) error {
+	diffs := &compatibilityErr{notAllowed: notAllowedChanges}
+	for location, prevSchema := range prevSchemaMap {
+		currSchema := currentSchemaMap[location]
+		executeSchemaCompareCheck(prevSchema, currSchema, diffs, schemaCompareFuncs)
+	}
+	for _, currSchema := range currentSchemaMap {
+		for _, schemaCheck := range schemaChecks {
+			schemaCheck(currSchema, diffs)
+		}
+	}
+	if diffs.isEmpty() {
+		return nil
+	}
+	return diffs
+}
+
+func CheckPropertyDeleted(prevSchema, currSchema *jsonschema.Schema, diffs *compatibilityErr) {
+	if prevSchema != nil && currSchema == nil {
+		diffs.add(schemaDeleted, prevSchema.Location, `property is removed`)
+	}
+}
+
+func CheckAdditionalProperties(schema *jsonschema.Schema, diffs *compatibilityErr) {
+	/*
+		enforcing open content model, in the future we can use existing additional properties schema to validate
+		new properties to ensure better adherence to schema.
+	*/
+	if schema.AdditionalProperties != nil {
+		property, ok := schema.AdditionalProperties.(bool)
+		if !ok || !property {
+			diffs.add(additionalPropertiesNotTrue, schema.Location, "additionalProperties need to be not defined or true for evaluation as an open content model")
+		}
+	}
+}
+
+func TypeCheckExecutor(spec TypeCheckSpec) SchemaCompareCheck {
+	return func(prevSchema, currSchema *jsonschema.Schema, diffs *compatibilityErr) {
+		if prevSchema == nil || currSchema == nil {
+			return
+		}
+		prevTypes := prevSchema.Types
+		currTypes := currSchema.Types
+		err := elementsMatch(prevTypes, currTypes) // special case of integer being allowed to changed to number is not respected due to additional code complexity
+		if err != nil {
+			diffs.add(subSchemaTypeModification, currSchema.Location, err.Error())
+			return
+		}
+		if len(currTypes) == 0 {
+			/*
+				types are not available for references and conditional schema types
+				ref/holder schema
+			*/
+			executeSchemaCompareCheck(prevSchema, currSchema, diffs, spec.emptyTypeChecks)
+			return
+		}
+		for _, schemaTypes := range prevTypes {
+			switch schemaTypes {
+			case "object":
+				executeSchemaCompareCheck(prevSchema, currSchema, diffs, spec.objectTypeChecks)
+			case "array":
+				// check item schema is same
+				executeSchemaCompareCheck(prevSchema, currSchema, diffs, spec.arrayTypeChecks)
+			case "integer":
+				// check for validation conflicts
+			case "string":
+				// check validation conflicts
+			case "number":
+				// check validation conflicts
+			case "boolean":
+
+			case "null":
+
+			default:
+				logger.Logger.Warn(fmt.Sprintf("Unexpected type %s", schemaTypes))
+			}
+		}
+	}
+}

formats/json/compatibility_helper.go

@@ -0,0 +1,179 @@
+package json
+
+import (
+	"strings"
+
+	"github.com/santhosh-tekuri/jsonschema/v5"
+)
+
+func checkEnum(prevSchema, currSchema *jsonschema.Schema, diffs *compatibilityErr) {
+	prevEnum := prevSchema.Enum
+	currEnum := currSchema.Enum
+	if prevEnum == nil && currEnum != nil {
+		diffs.add(enumCreation, currSchema.Location, "enum values added to existing non enum values")
+	}
+	if prevEnum != nil && currEnum == nil {
+		diffs.add(enumDeletion, currSchema.Location, "enum was deleted")
+	}
+	if prevEnum != nil && currEnum != nil {
+		if !isSubset(currEnum, prevEnum) {
+			diffs.add(enumElementDeletion, currSchema.Location, "enum property was deleted")
+		}
+	}
+}
+
+func checkRef(prevSchema, currSchema *jsonschema.Schema, diffs *compatibilityErr) {
+	if prevSchema.Ref != nil && currSchema.Ref != nil && prevSchema.Ref.Location != currSchema.Ref.Location { // check if prev and curr schema location are equivalent
+		diffs.add(refChanged, currSchema.Location, "ref for schema has been changed")
+	}
+	if prevSchema.Ref != nil && currSchema.Ref == nil {
+		diffs.add(refChanged, currSchema.Location, "ref for schema has been removed")
+	}
+	if prevSchema.Ref == nil && currSchema.Ref != nil {
+		diffs.add(refChanged, currSchema.Location, "ref for schema has been added")
+	}
+}
+
+func checkAnyOf(prevSchema, currSchema *jsonschema.Schema, diffs *compatibilityErr) {
+	prevAnyOf := prevSchema.AnyOf
+	currAnyOf := currSchema.AnyOf
+	if prevAnyOf != nil && currAnyOf != nil {
+		if len(prevAnyOf) < len(currAnyOf) {
+			diffs.add(anyOfElementAdded, currSchema.Location, "anyOf condition cannot have added elements")
+			return
+		}
+		if len(prevAnyOf) > len(currAnyOf) {
+			diffs.add(anyOfElementDeleted, currSchema.Location, "anyOf condition cannot have deleted elements")
+			return
+		}
+	}
+	if prevAnyOf == nil && currAnyOf != nil {
+		diffs.add(anyOfAdded, currSchema.Location, "anyOf condition cannot created during modification of schema")
+	}
+	if prevAnyOf != nil && currAnyOf == nil {
+		diffs.add(anyOfDeleted, currSchema.Location, "anyOf condition cannot be removed during modification of schema")
+	}
+}
+
+func checkOneOf(prevSchema, currSchema *jsonschema.Schema, diffs *compatibilityErr) {
+	prevOneOf := prevSchema.OneOf
+	currOneOf := currSchema.OneOf
+	if prevOneOf != nil && currOneOf != nil {
+		if len(prevOneOf) < len(currOneOf) {
+			diffs.add(oneOfElementAdded, currSchema.Location, "oneOf condition cannot have added elements")
+			return
+		}
+		if len(prevOneOf) > len(currOneOf) {
+			diffs.add(oneOfElementDeleted, currSchema.Location, "oneOf condition cannot have elements removed")
+		}
+	}
+	if prevOneOf == nil && currOneOf != nil {
+		diffs.add(oneOfAdded, currSchema.Location, "oneOf condition cannot created during modification of schema")
+	}
+	if prevOneOf != nil && currOneOf == nil {
+		diffs.add(oneOfDeleted, currSchema.Location, "oneOf condition cannot be removed during modification of schema")
+	}
+}
+
+func checkAllOf(prevSchema, currSchema *jsonschema.Schema, diffs *compatibilityErr) {
+	prevAllOf := prevSchema.AllOf
+	currAllOf := currSchema.AllOf
+	if prevAllOf != nil && currAllOf != nil {
+		if len(prevAllOf) != len(currAllOf) {
+			diffs.add(allOfModified, currSchema.Location, "allOf condition cannot be modified")
+			return
+		}
+	}
+	if prevAllOf == nil && currAllOf != nil {
+		diffs.add(allOfModified, currSchema.Location, "allOf condition cannot created during modification of schema")
+	}
+	if prevAllOf != nil && currAllOf == nil {
+		diffs.add(allOfModified, currSchema.Location, "allOf condition cannot be removed during modification of schema")
+	}
+}
+
+func checkItemSchema(prevSchema, currSchema *jsonschema.Schema, diffs *compatibilityErr) {
+	//  check index based schemas
+	if prevSchema.Draft == jsonschema.Draft2020 {
+		prevItems := prevSchema.PrefixItems
+		currItems := currSchema.PrefixItems
+		if prevItems != nil && currItems != nil {
+			if len(prevItems) != len(currItems) {
+				diffs.add(itemSchemaModification, currSchema.Location, "prev prefix items contains %d elements, current contains %d", len(prevItems), len(currItems))
+			}
+		}
+		if prevItems == nil && currItems != nil {
+			diffs.add(itemSchemaModification, currSchema.Location, "prev prefix items is absent, current contains %d", len(currItems))
+		}
+		if prevItems != nil && currItems == nil {
+			diffs.add(itemSchemaModification, currSchema.Location, "prev prefix items contains %d elements, current contains absent", len(prevItems))
+		}
+	} else {
+		prevItems := getItems(prevSchema)
+		currItems := getItems(currSchema)
+		if len(prevItems) != len(currItems) {
+			diffs.add(itemSchemaModification, currSchema.Location, "prev items contains %d elements, current contains %d", len(prevItems), len(currItems))
+		}
+	}
+}
+
+func checkRestOfItemsSchema(prevSchema, currSchema *jsonschema.Schema, diffs *compatibilityErr) {
+	var prevItem, currItem *jsonschema.Schema
+	var ok bool
+	// check schema for remaining array elements
+	if prevSchema.Draft == jsonschema.Draft2020 {
+		prevItem = prevSchema.Items2020
+		currItem = currSchema.Items2020
+	} else {
+		if prevSchema.AdditionalItems != nil {
+			prevItem, ok = prevSchema.AdditionalItems.(*jsonschema.Schema)
+			if !ok { // prev schema additional Items is boolean value
+				if prevSchema.AdditionalItems != currSchema.AdditionalItems {
+					// curr schema additional items is not equivalent
+					diffs.add(itemSchemaModification, prevSchema.Location, "the value of additional items has changed")
+				}
+				return // since both cases equal and non equal have been evaluated.
+			}
+		}
+		if currSchema.AdditionalItems != nil {
+			currItem, ok = currSchema.AdditionalItems.(*jsonschema.Schema)
+			if !ok { // curr schema is boolean
+				if prevSchema.AdditionalItems == nil {
+					diffs.add(itemSchemaAddition, prevSchema.Location, "additional items has been set, changes are not allowed to additional items")
+				} else if prevSchema.AdditionalItems != currSchema.AdditionalItems {
+					diffs.add(itemSchemaModification, prevSchema.Location, "additional items has been modified, changes are not allowed")
+				}
+				return
+			}
+		}
+	}
+	if prevItem == nil && currItem != nil {
+		diffs.add(itemSchemaAddition, currItem.Location, "item schema cannot be added in schema changes")
+	} else if prevItem != nil && currItem == nil {
+		diffs.add(itemsSchemaDeletion, prevItem.Location, "items schema cannot be deleted in modification changes")
+	}
+}
+
+func checkPropertyAddition(prevSchema, currSchema *jsonschema.Schema, diffs *compatibilityErr) {
+	prevProperties := getKeys(prevSchema.Properties)
+	currProperties := getKeys(currSchema.Properties)
+	addedKeys := getDiffernce(currProperties, prevProperties)
+	if len(addedKeys) > 0 {
+		diffs.add(propertyAddition, currSchema.Location, "added keys: %s", strings.Join(addedKeys, ","))
+	}
+}
+
+func checkRequiredProperties(prevSchema, currSchema *jsonschema.Schema, diffs *compatibilityErr) {
+	prevRequiredProperties := prevSchema.Required
+	currReqiredProperties := currSchema.Required
+	err := elementsMatch(prevRequiredProperties, currReqiredProperties)
+	if err != nil {
+		diffs.add(requiredFieldChanged, currSchema.Location, err.Error())
+	}
+}
+
+func executeSchemaCompareCheck(prev, curr *jsonschema.Schema, diffs *compatibilityErr, checks []SchemaCompareCheck) {
+	for _, check := range checks {
+		check(prev, curr, diffs)
+	}
+}