Skip to content

Commit

Permalink
Merge pull request #64 from flanksource/fix/failing-to-output-files-f…
Browse files Browse the repository at this point in the history
…or-windows

fix: normalises data for GetUnstructuredObjects to UTF-8 with LF endings
  • Loading branch information
moshloop authored Aug 10, 2021
2 parents e01dd0f + 3b4c346 commit 75c1fd5
Show file tree
Hide file tree
Showing 12 changed files with 626 additions and 77 deletions.
63 changes: 63 additions & 0 deletions .github/workflows/e2e.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
name: Karina upstream

on:
pull_request:
branches: [ master, main ]

jobs:
test:
runs-on: ubuntu-latest
env:
FILE_CHANGES_TO_CORE_ARTIFACTS: true
strategy:
fail-fast: false
matrix:
k8s:
- v1.18.6
suite:
- minimal
steps:
- uses: actions/checkout@v2
with:
path: kommons
- uses: actions/checkout@v2
with:
#Using Main branch of Karina Repository
repository: flanksource/karina
path: karina
- uses: actions/setup-go@v2
with:
go-version: '1.16.5' # The Go version to download (if necessary) and use.
- run: go version
- run: |
cd karina
module=$(cat go.mod | grep github.com/flanksource/kommons | awk '{print $2}')
go mod edit -replace github.com/flanksource/kommons@$module=$GITHUB_WORKSPACE/kommons
cat go.mod
go mod tidy
name: replace Module
- name: Test Make Linux
working-directory: ./karina
run: make linux

- name: Run e2e testing script
id: e2e
working-directory: ./karina
env:
SUITE: ${{ matrix.suite }}
KUBERNETES_VERSION: ${{matrix.k8s}}
BUILD: test (${{matrix.k8s}}, ${{ matrix.suite }})
ADDITIONAL_CONFIG: -c test/hosted-tests.yaml
run: ./test/test.sh
- name: Upload test results
uses: actions/upload-artifact@v2
with:
if-no-files-found: ignore
name: test-results-${{matrix.k8s}}-${{matrix.suite}}
path: test-results/
- name: Upload snapshots
uses: actions/upload-artifact@v2
with:
if-no-files-found: ignore
name: snapshot-${{matrix.k8s}}-${{matrix.suite}}
path: artifacts/snapshot.zip
1 change: 1 addition & 0 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ on:
push:
branches:
- master
- main
jobs:
semantic-release:
runs-on: ubuntu-20.04
Expand Down
85 changes: 22 additions & 63 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
@@ -1,63 +1,22 @@
name: Karina upstream

on:
pull_request:
branches: [ master ]

jobs:
Test:
runs-on: ubuntu-latest
env:
FILE_CHANGES_TO_CORE_ARTIFACTS: true
strategy:
fail-fast: false
matrix:
k8s:
- v1.18.6
suite:
- minimal
steps:
- uses: actions/checkout@v2
with:
path: kommons
- uses: actions/checkout@v2
with:
#Using Main branch of Karina Repository
repository: flanksource/karina
path: karina
- uses: actions/setup-go@v2
with:
go-version: '1.16.5' # The Go version to download (if necessary) and use.
- run: go version
- run: |
cd karina
module=$(cat go.mod | grep github.com/flanksource/kommons | awk '{print $2}')
go mod edit -replace github.com/flanksource/kommons@$module=$GITHUB_WORKSPACE/kommons
cat go.mod
go mod tidy
name: replace Module
- name: Test Make Linux
working-directory: ./karina
run: make linux

- name: Run e2e testing script
id: e2e
working-directory: ./karina
env:
SUITE: ${{ matrix.suite }}
KUBERNETES_VERSION: ${{matrix.k8s}}
BUILD: test (${{matrix.k8s}}, ${{ matrix.suite }})
ADDITIONAL_CONFIG: -c test/hosted-tests.yaml
run: ./test/test.sh
- name: Upload test results
uses: actions/upload-artifact@v2
with:
if-no-files-found: ignore
name: test-results-${{matrix.k8s}}-${{matrix.suite}}
path: test-results/
- name: Upload snapshots
uses: actions/upload-artifact@v2
with:
if-no-files-found: ignore
name: snapshot-${{matrix.k8s}}-${{matrix.suite}}
path: artifacts/snapshot.zip
name: Testing

on:
pull_request:
branches: [ master, main ]

jobs:
gotest:
strategy:
matrix:
go-version: [1.16.x]
platform: [ubuntu-latest]
runs-on: ${{ matrix.platform }}
steps:
- name: Install Go
uses: actions/setup-go@v2
with:
go-version: ${{ matrix.go-version }}
- name: Checkout code
uses: actions/checkout@v2
- name: Test
run: make test
8 changes: 8 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -25,3 +25,11 @@ CONTROLLER_GEN=$(GOBIN)/controller-gen
else
CONTROLLER_GEN=$(shell which controller-gen)
endif

.PHONY: test
test:
go test ./... -test.v

.PHONY: fmt
fmt:
go fmt ./...
4 changes: 4 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,17 @@ go 1.16

require (
github.com/AlekSi/pointer v1.1.0
github.com/TomOnTime/utfutil v0.0.0-20210710122150-437f72b26edf
github.com/flanksource/commons v1.5.6
github.com/hairyhenderson/gomplate/v3 v3.6.0
github.com/mitchellh/mapstructure v1.3.3
github.com/pkg/errors v0.9.1
github.com/sergi/go-diff v1.0.0
github.com/sirupsen/logrus v1.7.0
github.com/stretchr/testify v1.7.0
github.com/tidwall/gjson v1.6.7
go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489
golang.org/x/text v0.3.6
google.golang.org/grpc v1.27.1
gopkg.in/flanksource/yaml.v3 v3.1.1
k8s.io/api v0.20.4
Expand All @@ -26,5 +29,6 @@ require (

replace (
github.com/Azure/go-autorest => github.com/Azure/go-autorest v14.2.0+incompatible
github.com/go-logr/logr => github.com/go-logr/logr v0.4.0
k8s.io/client-go => k8s.io/client-go v0.20.4
)
13 changes: 8 additions & 5 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,8 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
github.com/Shopify/ejson v1.2.1 h1:Dx0Ipn0mUgrZlzIa5oIUrH0rdSmBOyod/UJmQQK1KHo=
github.com/Shopify/ejson v1.2.1/go.mod h1:J8cw5GOA0l/aMOPp+uDfwNYVbeqIaBhzRkv1+76UCvk=
github.com/TomOnTime/utfutil v0.0.0-20210710122150-437f72b26edf h1:+GdVyvpzTy3UFAS1+hbTqm9Mk0U1Xrocm28s/E2GWz0=
github.com/TomOnTime/utfutil v0.0.0-20210710122150-437f72b26edf/go.mod h1:FiuynIwe98RFhWI8nZ0dnsldPVsBy9rHH1hn2WYwme4=
github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM=
github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8=
Expand Down Expand Up @@ -198,9 +200,8 @@ github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2
github.com/go-ldap/ldap v3.0.2+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp40uXYvFoEVrNEPqRc=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
github.com/go-logr/logr v0.2.0 h1:QvGt2nLcHH0WK9orKa+ppBPAxREcH364nPUedEpK0TY=
github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc=
github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w=
github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
Expand Down Expand Up @@ -593,8 +594,9 @@ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoH
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/tidwall/gjson v1.6.7 h1:Mb1M9HZCRWEcXQ8ieJo7auYyyiSux6w9XN3AdTpxJrE=
github.com/tidwall/gjson v1.6.7/go.mod h1:zeFuBCIqD4sN/gmqBzZ4j7Jd6UcA2Fc56x7QFsv+8fI=
Expand Down Expand Up @@ -795,8 +797,9 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3
golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc=
golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
Expand Down
26 changes: 24 additions & 2 deletions kustomize/kustomize.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ import (
"strings"
"sync"

"github.com/TomOnTime/utfutil"
"github.com/flanksource/commons/logger"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
Expand Down Expand Up @@ -281,9 +283,13 @@ func (km *Manager) Kustomize(namespace string, objects ...runtime.Object) ([]run
}

func GetUnstructuredObjects(data []byte) ([]runtime.Object, error) {
utfData, err := BytesToUtf8Lf(data)
if err != nil {
return nil, fmt.Errorf("error converting to UTF %v", err)
}
var items []runtime.Object
re := regexp.MustCompile("(?m)^---\\n")
for _, chunk := range re.Split(string(data), -1) {
re := regexp.MustCompile(`(?m)^---\n`)
for _, chunk := range re.Split(utfData, -1) {
if strings.TrimSpace(chunk) == "" {
continue
}
Expand All @@ -300,3 +306,19 @@ func GetUnstructuredObjects(data []byte) ([]runtime.Object, error) {

return items, nil
}

func BytesToUtf8Lf(file []byte) (string, error) {
decoded := utfutil.BytesReader(file, utfutil.UTF8)
buf := new(bytes.Buffer)
_, err := buf.ReadFrom(decoded)
if err != nil {
logger.Errorf("error reading from buffer: %v", err)
return "", err
}
val := buf.Bytes()
// replace \r with \n -> solves for Mac but leaves \n\n for Windows
val = bytes.ReplaceAll(val, []byte{13}, []byte{10})
// replace \n\n with \n
val = bytes.ReplaceAll(val, []byte{10, 10}, []byte{10})
return string(val), nil
}
Loading

0 comments on commit 75c1fd5

Please sign in to comment.