From 8e6c80ac74248c1c65cdda3f0179894274558876 Mon Sep 17 00:00:00 2001 From: JoshKCarroll Date: Thu, 12 Oct 2017 15:04:47 -0400 Subject: [PATCH] #62 Add Delete Transform (#63) * Initial setup for delete transform * Finish Delete transform and add more tests --- README.md | 34 +++++++++ kazaam.go | 1 + kazaam_int_test.go | 20 ++++++ kazaam_test.go | 2 +- transform/delete.go | 31 ++++++++ transform/delete_test.go | 150 +++++++++++++++++++++++++++++++++++++++ transform/util.go | 44 ++++++++++++ 7 files changed, 281 insertions(+), 1 deletion(-) create mode 100644 transform/delete.go create mode 100644 transform/delete_test.go diff --git a/README.md b/README.md index 276b271..39d5c12 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,7 @@ Kazaam currently supports the following transforms: - uuid - default - pass +- delete ### Shift The shift transform is the current Kazaam workhorse used for remapping of fields. @@ -357,6 +358,39 @@ A default transform provides the ability to set a key's value explicitly. For ex would ensure that the output JSON message includes `{"type": "message"}`. +### Delete +A delete transform provides the ability to delete keys in place. +```javascript +{ + "operation": "delete", + "spec": { + "paths": ["doc.uid", "doc.guidObjects[1]"] + } +} +``` + +executed on a json message with format +```javascript +{ + "doc": { + "uid": 12345, + "guid": ["guid0", "guid2", "guid4"], + "guidObjects": [{"id": "guid0"}, {"id": "guid2"}, {"id": "guid4"}] + } +} +``` + +would result in +```javascript +{ + "doc": { + "guid": ["guid0", "guid2", "guid4"], + "guidObjects": [{"id": "guid0"}, {"id": "guid4"}] + } +} +``` + + ### Pass A pass transform, as the name implies, passes the input data unchanged to the output. This is used internally when a null transform spec is specified, but may also be useful for testing. diff --git a/kazaam.go b/kazaam.go index 9b20532..c44d404 100644 --- a/kazaam.go +++ b/kazaam.go @@ -31,6 +31,7 @@ func init() { "shift": transform.Shift, "extract": transform.Extract, "default": transform.Default, + "delete": transform.Delete, "concat": transform.Concat, "coalesce": transform.Coalesce, "timestamp": transform.Timestamp, diff --git a/kazaam_int_test.go b/kazaam_int_test.go index 31c10a0..8dd95c5 100644 --- a/kazaam_int_test.go +++ b/kazaam_int_test.go @@ -415,3 +415,23 @@ func TestKazaamTransformTwoOpWithOverRequire(t *testing.T) { t.FailNow() } } + +func TestKazaamTransformDelete(t *testing.T) { + spec := `[{ + "operation": "delete", + "spec": {"paths": ["doc.uid", "doc.guidObjects[1]"]} + }]` + jsonIn := `{"doc":{"uid":12345,"guid":["guid0","guid2","guid4"],"guidObjects":[{"id":"guid0"},{"id":"guid2"},{"id":"guid4"}]}}` + jsonOut := `{"doc":{"guid":["guid0","guid2","guid4"],"guidObjects":[{"id":"guid0"},{"id":"guid4"}]}}` + + kazaamTransform, _ := kazaam.NewKazaam(spec) + kazaamOut, _ := kazaamTransform.TransformJSONStringToString(jsonIn) + areEqual, _ := checkJSONStringsEqual(kazaamOut, jsonOut) + + if !areEqual { + t.Error("Transformed data does not match expectation.") + t.Log("Expected: ", jsonOut) + t.Log("Actual: ", kazaamOut) + t.FailNow() + } +} diff --git a/kazaam_test.go b/kazaam_test.go index ab51f7d..261e636 100644 --- a/kazaam_test.go +++ b/kazaam_test.go @@ -38,7 +38,7 @@ func TestReregisterKazaamTransform(t *testing.T) { } func TestDefaultTransformsSetCardinarily(t *testing.T) { - if len(validSpecTypes) != 8 { + if len(validSpecTypes) != 9 { t.Error("Unexpected number of default transforms. Missing tests?") } } diff --git a/transform/delete.go b/transform/delete.go new file mode 100644 index 0000000..acb791e --- /dev/null +++ b/transform/delete.go @@ -0,0 +1,31 @@ +package transform + +import ( + "fmt" +) + +// Delete deletes keys in-place from the provided data if they exist +// keys are specified in an array under "keys" in the spec. +func Delete(spec *Config, data []byte) ([]byte, error) { + paths, pathsOk := (*spec.Spec)["paths"] + if !pathsOk { + return nil, SpecError("Unable to get paths to delete") + } + pathSlice, sliceOk := paths.([]interface{}) + if !sliceOk { + return nil, SpecError(fmt.Sprintf("paths should be a slice of strings: %v", paths)) + } + for _, pItem := range pathSlice { + path, ok := pItem.(string) + if !ok { + return nil, SpecError(fmt.Sprintf("Error processing %v: path should be a string", pItem)) + } + + var err error + data, err = delJSONRaw(data, path, spec.Require) + if err != nil { + return nil, err + } + } + return data, nil +} diff --git a/transform/delete_test.go b/transform/delete_test.go new file mode 100644 index 0000000..5516642 --- /dev/null +++ b/transform/delete_test.go @@ -0,0 +1,150 @@ +package transform + +import "testing" + +func TestDelete(t *testing.T) { + spec := `{"paths": ["rating.example"]}` + jsonOut := `{"rating":{"primary":{"value":3}}}` + + cfg := getConfig(spec, false) + kazaamOut, err := getTransformTestWrapper(Delete, cfg, testJSONInput) + + if err != nil { + t.Error("Error in transform (simplejson).") + t.Log("Error: ", err.Error()) + t.FailNow() + } + + areEqual, _ := checkJSONBytesEqual(kazaamOut, []byte(jsonOut)) + if !areEqual { + t.Error("Transformed data does not match expectation.") + t.Log("Expected: ", jsonOut) + t.Log("Actual: ", string(kazaamOut)) + t.FailNow() + } +} + +func TestDeleteSpecErrorNoPathsKey(t *testing.T) { + spec := `{"pathz": ["a.path"]}` + expectedErr := "Unable to get paths to delete" + + cfg := getConfig(spec, false) + _, err := getTransformTestWrapper(Delete, cfg, testJSONInput) + + if err == nil { + t.Error("Should have generated error for invalid paths") + t.Log("Spec: ", spec) + t.FailNow() + } + e, ok := err.(SpecError) + if !ok { + t.Error("Unexpected error type") + t.FailNow() + } + + if e.Error() != expectedErr { + t.Error("Unexpected error details") + t.Log("Expected: ", expectedErr) + t.Log("Actual: ", e.Error()) + t.FailNow() + } +} + +func TestDeleteSpecErrorInvalidPaths(t *testing.T) { + spec := `{"paths": false}` + expectedErr := "paths should be a slice of strings: false" + + cfg := getConfig(spec, false) + _, err := getTransformTestWrapper(Delete, cfg, testJSONInput) + + if err == nil { + t.Error("Should have generated error for invalid paths") + t.Log("Spec: ", spec) + t.FailNow() + } + e, ok := err.(SpecError) + if !ok { + t.Error("Unexpected error type") + t.FailNow() + } + + if e.Error() != expectedErr { + t.Error("Unexpected error details") + t.Log("Expected: ", expectedErr) + t.Log("Actual: ", e.Error()) + t.FailNow() + } +} + +func TestDeleteSpecErrorInvalidPathItem(t *testing.T) { + spec := `{"paths": ["foo", 42]}` + expectedErr := "Error processing 42: path should be a string" + + cfg := getConfig(spec, false) + _, err := getTransformTestWrapper(Delete, cfg, testJSONInput) + + if err == nil { + t.Error("Should have generated error for invalid paths") + t.Log("Spec: ", spec) + t.FailNow() + } + e, ok := err.(SpecError) + if !ok { + t.Error("Unexpected error type") + t.FailNow() + } + + if e.Error() != expectedErr { + t.Error("Unexpected error details") + t.Log("Expected: ", expectedErr) + t.Log("Actual: ", e.Error()) + t.FailNow() + } +} + +func TestDeleteSpecErrorWildcardNotSupported(t *testing.T) { + spec := `{"paths": ["ratings[*].value"]}` + jsonIn := `{"ratings: [{"value": 3, "user": "rick"}, {"value": 7, "user": "jerry"}]}` + expectedErr := "Array wildcard not supported for this operation." + + cfg := getConfig(spec, false) + _, err := getTransformTestWrapper(Delete, cfg, jsonIn) + + if err == nil { + t.Error("Should have generated error for invalid paths") + t.Log("Spec: ", spec) + t.FailNow() + } + e, ok := err.(SpecError) + if !ok { + t.Error("Unexpected error type") + t.FailNow() + } + + if e.Error() != expectedErr { + t.Error("Unexpected error details") + t.Log("Expected: ", expectedErr) + t.Log("Actual: ", e.Error()) + t.FailNow() + } +} + +func TestDeleteWithRequire(t *testing.T) { + spec := `{"paths": ["rating.examplez"]}` + + cfg := getConfig(spec, true) + _, err := getTransformTestWrapper(Delete, cfg, testJSONInput) + + if err == nil { + t.Error("Should have generated error for invalid paths") + t.Log("Spec: ", spec) + t.FailNow() + } + _, ok := err.(RequireError) + if !ok { + t.Error("Unexpected error type") + t.Error(err.Error()) + t.FailNow() + } + +} diff --git a/transform/util.go b/transform/util.go index dbae55a..b237452 100644 --- a/transform/util.go +++ b/transform/util.go @@ -208,6 +208,50 @@ func setJSONRaw(data, out []byte, path string) ([]byte, error) { return data, nil } +// delJSONRaw deletes the value at a path and handles array indexing +func delJSONRaw(data []byte, path string, pathRequired bool) ([]byte, error) { + var err error + splitPath := strings.Split(path, ".") + numOfInserts := 0 + + for element, k := range splitPath { + arrayRefs := jsonPathRe.FindAllStringSubmatch(k, -1) + if arrayRefs != nil && len(arrayRefs) > 0 { + objKey := arrayRefs[0][1] // the key + arrayKeyStr := arrayRefs[0][2] // the array index + err = validateArrayKeyString(arrayKeyStr) + if err != nil { + return nil, err + } + + // not currently supported + if arrayKeyStr == "*" { + return nil, SpecError("Array wildcard not supported for this operation.") + } + + // if not a wildcard then piece that path back together with the + // array index as an entry in the splitPath slice + splitPath = makePathWithIndex(arrayKeyStr, objKey, splitPath, element+numOfInserts) + numOfInserts++ + } else { + // no array reference, good to go + continue + } + } + + if pathRequired { + _, _, _, err = jsonparser.Get(data, splitPath...) + if err == jsonparser.KeyPathNotFoundError { + return nil, NonExistentPath + } else if err != nil { + return nil, err + } + } + + data = jsonparser.Delete(data, splitPath...) + return data, nil +} + // validateArrayKeyString is a helper function to make sure the array index is // legal func validateArrayKeyString(arrayKeyStr string) error {