Skip to content

Commit

Permalink
fixup! fix: normalises data for GetUnstructuredObjects to UTF-8 with …
Browse files Browse the repository at this point in the history 
…LF endings
  • Loading branch information
Saul committed Aug 10, 2021
1 parent ade8ce0 commit 3b4c346
Show file tree
Hide file tree
Showing 4 changed files with 47 additions and 45 deletions.
4 changes: 4 additions & 0 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,3 +29,7 @@ endif
.PHONY: test
test:
go test ./... -test.v

.PHONY: fmt
fmt:
go fmt ./...
28 changes: 14 additions & 14 deletions kustomize/kustomize.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,27 +17,28 @@ package kustomize
import (
"bytes"
"fmt"
"io/ioutil"
"path/filepath"
"regexp"
osruntime "runtime"
"strings"
"sync"

"github.com/TomOnTime/utfutil"
"github.com/flanksource/commons/logger"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"io/ioutil"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
yamlutil "k8s.io/apimachinery/pkg/util/yaml"
"k8s.io/cli-runtime/pkg/kustomize"
"path/filepath"
"regexp"
osruntime "runtime"
"sigs.k8s.io/kustomize/pkg/constants"
"sigs.k8s.io/kustomize/pkg/fs"
"sigs.k8s.io/kustomize/pkg/ifc"
"sigs.k8s.io/kustomize/pkg/loader"
"sigs.k8s.io/kustomize/pkg/patch"
"sigs.k8s.io/kustomize/pkg/types"
"sigs.k8s.io/yaml"
"strings"
"sync"
)

// Manager define a manager that allow access to kustomize capabilities
Expand Down Expand Up @@ -283,11 +284,11 @@ func (km *Manager) Kustomize(namespace string, objects ...runtime.Object) ([]run

func GetUnstructuredObjects(data []byte) ([]runtime.Object, error) {
utfData, err := BytesToUtf8Lf(data)
if err != nil {
return nil, fmt.Errorf("error converting to UTF %s", err)
if err != nil {
return nil, fmt.Errorf("error converting to UTF %v", err)
}
var items []runtime.Object
re := regexp.MustCompile("(?m)^---\\n")
re := regexp.MustCompile(`(?m)^---\n`)
for _, chunk := range re.Split(utfData, -1) {
if strings.TrimSpace(chunk) == "" {
continue
Expand All @@ -309,16 +310,15 @@ func GetUnstructuredObjects(data []byte) ([]runtime.Object, error) {
func BytesToUtf8Lf(file []byte) (string, error) {
decoded := utfutil.BytesReader(file, utfutil.UTF8)
buf := new(bytes.Buffer)
bytesRead, err := buf.ReadFrom(decoded)
_, err := buf.ReadFrom(decoded)
if err != nil {
logger.Errorf("error reading from buffer. bytesRead %v. err: %v", bytesRead, err)
logger.Errorf("error reading from buffer: %v", err)
return "", err
}
val := buf.Bytes()
// replace \r with \n -> solves for Mac but leaves \n\n for Windows
val = bytes.Replace(val, []byte{13}, []byte{10}, -1)
val = bytes.ReplaceAll(val, []byte{13}, []byte{10})
// replace \n\n with \n
val = bytes.Replace(val, []byte{10, 10}, []byte{10}, -1)
val = bytes.ReplaceAll(val, []byte{10, 10}, []byte{10})
return string(val), nil
}

58 changes: 28 additions & 30 deletions kustomize/kustomize_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ import (
"testing"
)


type UTFWriteCloser interface {
Write(p []byte) (n int, err error)
Close() error
Expand Down Expand Up @@ -68,13 +67,13 @@ func bytesWriter(b *bytes.Buffer, d utfutil.EncodingHint) io.Writer {

func bytesToCrlfOrCr(file []byte, os string, hint utfutil.EncodingHint) (string, error) {
val := file
val = bytes.Replace(val, []byte{13}, []byte{10}, -1)
val = bytes.Replace(val, []byte{10, 10}, []byte{10}, -1)
val = bytes.ReplaceAll(val, []byte{13}, []byte{10})
val = bytes.ReplaceAll(val, []byte{10, 10}, []byte{10})
switch os {
case "macos":
val = bytes.Replace(val, []byte{10}, []byte{13}, -1)
val = bytes.ReplaceAll(val, []byte{10}, []byte{13})
default:
val = bytes.Replace(val, []byte{10}, []byte{13, 10}, -1)
val = bytes.ReplaceAll(val, []byte{10}, []byte{13, 10})
}
var buf bytes.Buffer
writer := bytesWriter(&buf, hint)
Expand All @@ -101,7 +100,6 @@ func getFileBuffer(filePath string) ([]byte, error) {
return buf, nil
}


func TestBytesToUtf8Lf(t *testing.T) {

// If tests are breaking, first check that the file read is UTF-8 with LF endings
Expand All @@ -121,7 +119,7 @@ func TestBytesToUtf8Lf(t *testing.T) {
t.Errorf("error setting up test TestBytesToUtf8Lf: %v", err)
return
}
utf16BeCrMac, err := bytesToCrlfOrCr(data, "macos", utfutil.UTF16BE)
utf16BeCrMac, err := bytesToCrlfOrCr(data, "macos", utfutil.UTF16BE)
if err != nil {
t.Errorf("error setting up test TestBytesToUtf8Lf: %v", err)
return
Expand Down Expand Up @@ -204,13 +202,13 @@ func TestGetUnstructuredObjects(t *testing.T) {
Object: map[string]interface{}{
"apiVersion": "v1",
"data": map[string]interface{}{
"logging.level.org.springframework": "DEBUG",
"logging.level.org.springframework": "DEBUG",
"logging.level.org.springframework.web": "INFO",
"some-key": "value-from-spring",
"some-key": "value-from-spring",
},
"kind": "ConfigMap",
"metadata": map[string]interface{}{
"name": "spring-defaults-spring",
"name": "spring-defaults-spring",
"namespace": "default",
},
},
Expand All @@ -220,12 +218,12 @@ func TestGetUnstructuredObjects(t *testing.T) {
Object: map[string]interface{}{
"apiVersion": "v1",
"data": map[string]interface{}{
"application.properties": "some-key=new-value\nnew-key=diff-value\n",
"application.properties": "some-key=new-value\nnew-key=diff-value\n",
},
"kind": "ConfigMap",
"metadata": map[string]interface{}{
"name": "sample",
"namespace": "default",
"name": "sample",
"namespace": "default",
},
},
}
Expand All @@ -246,7 +244,7 @@ func TestGetUnstructuredObjects(t *testing.T) {
t.Errorf("error setting up test TestBytesToUtf8Lf: %v", err)
return
}
utf16BeCrMac, err := bytesToCrlfOrCr(data, "macos", utfutil.UTF16BE)
utf16BeCrMac, err := bytesToCrlfOrCr(data, "macos", utfutil.UTF16BE)
if err != nil {
t.Errorf("error setting up test TestBytesToUtf8Lf: %v", err)
return
Expand All @@ -257,61 +255,61 @@ func TestGetUnstructuredObjects(t *testing.T) {
return
}

items := []runtime.Object{ itemOne, itemTwo }
items := []runtime.Object{itemOne, itemTwo}
type args struct {
data []byte
}
tests := []struct {
name string
args args
want []runtime.Object
name string
args args
want []runtime.Object
testIndexes []int
wantErr bool
wantErr bool
}{
{
name: "f(data)->[]runtime.Object{ ...items }",
args: args{
data: data,
},
want: items,
want: items,
testIndexes: []int{0, 1},
wantErr: false,
wantErr: false,
},
{
name: "f(utf16LeCrMac)->[]runtime.Object{ ...items }",
args: args{
data: []byte(utf16LeCrMac),
},
want: items,
want: items,
testIndexes: []int{0, 1},
wantErr: false,
wantErr: false,
},
{
name: "f(utf16BeCrMac)->[]runtime.Object{ ...items }",
args: args{
data: []byte(utf16BeCrMac),
},
want: items,
want: items,
testIndexes: []int{0, 1},
wantErr: false,
wantErr: false,
},
{
name: "f(utf16LeCrlfDefault)->[]runtime.Object{ ...items }",
args: args{
data: []byte(utf16LeCrlfDefault),
},
want: items,
want: items,
testIndexes: []int{0, 1},
wantErr: false,
wantErr: false,
},
{
name: "f(utf16BeCrlfDefault)->[]runtime.Object{ ...items }",
args: args{
data: []byte(utf16BeCrlfDefault),
},
want: items,
want: items,
testIndexes: []int{0, 1},
wantErr: false,
wantErr: false,
},
}
for _, tt := range tests {
Expand All @@ -329,4 +327,4 @@ func TestGetUnstructuredObjects(t *testing.T) {
}
})
}
}
}
2 changes: 1 addition & 1 deletion utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -746,4 +746,4 @@ func HasTaint(node v1.Node, name string) bool {
}
}
return false
}
}

0 comments on commit 3b4c346

Please sign in to comment.