diff --git a/TUTORIAL.md b/TUTORIAL.md index dcb70362..026ca138 100644 --- a/TUTORIAL.md +++ b/TUTORIAL.md @@ -262,13 +262,13 @@ The `puccini-clout scriptlet exec` command can also execute scriptlets that are embedded in the Clout. Let's use a scriptlet that creates an HTML page that visualizes the topology: - puccini-clout scriptlet exec assets/profiles/common/1.0/js/visualize.js clout.yaml --output=tosca.html + puccini-clout scriptlet exec assets/tosca/profiles/common/1.0/js/visualize.js clout.yaml --output=tosca.html xdg-open tosca.html Note another shortcut for `puccini-tosca compile`: you can use the `--exec` flag to execute scriptlets right after compilation, thus skipping the Clout intermediary: - puccini-tosca compile examples/tosca/requirements-and-capabilities.yaml --exec=assets/profiles/common/1.0/js/visualize.js + puccini-tosca compile examples/tosca/requirements-and-capabilities.yaml --exec=assets/tosca/profiles/common/1.0/js/visualize.js See [here](puccini-clout/) for more information about the `puccini-clout` tool. diff --git a/clout/js/clout-api.go b/clout/js/clout-api.go index 42b052a0..c6f3dd46 100644 --- a/clout/js/clout-api.go +++ b/clout/js/clout-api.go @@ -9,6 +9,7 @@ import ( "github.com/dop251/goja" "github.com/fxamacker/cbor/v2" "github.com/tliron/commonjs-goja" + "github.com/tliron/exturl" "github.com/tliron/go-ard" cloutpkg "github.com/tliron/puccini/clout" "github.com/vmihailenco/msgpack/v5" @@ -36,8 +37,14 @@ func (self *CloutAPI) Load(context contextpkg.Context, data any) (*CloutAPI, err var err error switch data_ := data.(type) { + case exturl.URL: + if clout, err = cloutpkg.Load(context, data_); err != nil { + return nil, err + } + case string: - if clout, err = cloutpkg.Load(context, data_, "", self.cloutContext.Context.URLContext); err != nil { + url := self.cloutContext.Context.URLContext.NewAnyOrFileURL(data_) + if clout, err = cloutpkg.Load(context, url); err != nil { return nil, err } @@ -62,6 +69,7 @@ func (self *CloutAPI) Call(scriptletName string, functionName string, arguments return executionContext.Call(scriptletName, functionName, arguments) } +// TODO: unused? func (self *CloutAPI) CallAll(function goja.FunctionCall) goja.Value { if len(function.Arguments) >= 2 { if scriptletBaseName, ok := function.Arguments[0].Export().(string); ok { diff --git a/clout/js/context.go b/clout/js/context.go index c4f8f3ba..fb6a28c0 100644 --- a/clout/js/context.go +++ b/clout/js/context.go @@ -98,5 +98,9 @@ func (self *Context) NewEnvironment(clout *cloutpkg.Clout, apis map[string]any) func (self *Context) Require(clout *cloutpkg.Clout, scriptletName string, apis map[string]any) (*goja.Object, error) { environment := self.NewEnvironment(clout, apis) - return environment.RequireID(scriptletName) + if r, err := environment.RequireID(scriptletName); err == nil { + return r, nil + } else { + return r, UnwrapException(err) + } } diff --git a/clout/js/puccini-api.go b/clout/js/puccini-api.go index 48ee00d4..d5a8c4d3 100644 --- a/clout/js/puccini-api.go +++ b/clout/js/puccini-api.go @@ -110,7 +110,7 @@ func (self *PucciniAPI) Write(data any, path string, dontOverwrite bool) { func (self *PucciniAPI) LoadString(url string) (string, error) { context := contextpkg.TODO() - if url_, err := self.context.URLContext.NewValidURL(context, url, nil); err == nil { + if url_, err := self.context.URLContext.NewValidAnyOrFileURL(context, url, nil); err == nil { return exturl.ReadString(context, url_) } else { return "", err diff --git a/clout/load.go b/clout/load.go index f5a7a0ad..8e4fb394 100644 --- a/clout/load.go +++ b/clout/load.go @@ -7,24 +7,11 @@ import ( "github.com/tliron/kutil/util" ) -func Load(context contextpkg.Context, url string, format string, urlContext *exturl.Context) (*Clout, error) { - var url_ exturl.URL - - var err error - if url != "" { - if url_, err = urlContext.NewValidURL(context, url, nil); err != nil { - return nil, err - } - } else { - if url_, err = urlContext.ReadToInternalURLFromStdin(context, format); err != nil { - return nil, err - } - } - - if reader, err := url_.Open(context); err == nil { +func Load(context contextpkg.Context, url exturl.URL) (*Clout, error) { + if reader, err := url.Open(context); err == nil { reader = util.NewContextualReadCloser(context, reader) defer reader.Close() - return Read(reader, url_.Format()) + return Read(reader, url.Format()) } else { return nil, err } diff --git a/go.mod b/go.mod index 4326c647..fadf95ff 100644 --- a/go.mod +++ b/go.mod @@ -3,16 +3,16 @@ module github.com/tliron/puccini go 1.21 require ( - github.com/dop251/goja v0.0.0-20230812105242-81d76064690d + github.com/dop251/goja v0.0.0-20230828202809-3dbe69dd2b8e github.com/fxamacker/cbor/v2 v2.5.0 github.com/klauspost/compress v1.16.7 github.com/klauspost/pgzip v1.2.6 github.com/segmentio/ksuid v1.0.4 github.com/spf13/cobra v1.7.0 - github.com/tliron/commonjs-goja v0.1.0 + github.com/tliron/commonjs-goja v0.1.2 github.com/tliron/commonlog v0.1.1 - github.com/tliron/exturl v0.2.9 - github.com/tliron/go-ard v0.1.3 + github.com/tliron/exturl v0.4.0 + github.com/tliron/go-ard v0.1.4 github.com/tliron/kutil v0.2.11 github.com/tliron/yamlkeys v1.3.6 github.com/vmihailenco/msgpack/v5 v5.3.5 diff --git a/go.sum b/go.sum index 96a0cbdd..762bf7b7 100644 --- a/go.sum +++ b/go.sum @@ -41,8 +41,8 @@ github.com/docker/docker v24.0.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bc github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryefjlVRqWqIhk/uXJp0A= github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0= github.com/dop251/goja v0.0.0-20211022113120-dc8c55024d06/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk= -github.com/dop251/goja v0.0.0-20230812105242-81d76064690d h1:9aaGwVf4q+kknu+mROAXUApJ1DoOwhE8dGj/XLBYzWg= -github.com/dop251/goja v0.0.0-20230812105242-81d76064690d/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4= +github.com/dop251/goja v0.0.0-20230828202809-3dbe69dd2b8e h1:UvQD6hTSfeM6hhTQ24Dlw2RppP05W7SWbWb6kubJAog= +github.com/dop251/goja v0.0.0-20230828202809-3dbe69dd2b8e/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4= github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y= github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM= github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819 h1:RIB4cRk+lBqKK3Oy0r2gRX4ui7tuhiZq2SuTtTCi0/0= @@ -166,14 +166,14 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/tliron/commonjs-goja v0.1.0 h1:BF1NitaiHl2RLxqjp9lGNXx2eiJtTKKk3UFpSEp3j5c= -github.com/tliron/commonjs-goja v0.1.0/go.mod h1:mgGnXjwqvLSzMqsfLpwFGKS5qz40b66rE97IWHLQUSA= +github.com/tliron/commonjs-goja v0.1.2 h1:DZj5x/WlkBPxPwg1DF+kYHkQpraxvzJ4UHUQHkvd/no= +github.com/tliron/commonjs-goja v0.1.2/go.mod h1:B6gZxxfXlRR7DEcRbDvktu+KRW9L1zcEpg2lJLzqzdk= github.com/tliron/commonlog v0.1.1 h1:bCkNKk+O5ciKBFxnjcMGLR79TGCq8hMSh5DfaeRJQHk= github.com/tliron/commonlog v0.1.1/go.mod h1:qswie0h44wu9XrQ8sGOtmOjVVgI5qbU4Dl4wbi1b/w0= -github.com/tliron/exturl v0.2.9 h1:7utvDHC9d1tqC/eHmiZvzklCn/ZRH7+/5QvjAU5+4Vc= -github.com/tliron/exturl v0.2.9/go.mod h1:FoKROnSga8WXPeDHQTFPTaQPAyeJTR3PVGhycvM2hhg= -github.com/tliron/go-ard v0.1.3 h1:E7M2Bpyb51wplLJGf5A4NqaOGoQqf2HuMwlmOGSi+Ig= -github.com/tliron/go-ard v0.1.3/go.mod h1:zhlpib5+3HdpBEfTyU+KZ6XskBf2az8up4ZgJyT9hVw= +github.com/tliron/exturl v0.4.0 h1:2lHilabGmaoux3oYyPMJxcAcuMevVbw8cZ3LL9qxYB4= +github.com/tliron/exturl v0.4.0/go.mod h1:OGgLAYa3qGYKmRu/RVjKnNvw4Va4lhCxKMxMbbdicSI= +github.com/tliron/go-ard v0.1.4 h1:NNtNdlX2Gxh14pIOd7xHBYIOws3XUCEIH4379Apag08= +github.com/tliron/go-ard v0.1.4/go.mod h1:OFSzMH/CINMYBmCUwEzUmWvNbnAMJJE9f2LARwo4G3w= github.com/tliron/kutil v0.2.11 h1:DtYxIcABJK1sdDn6F5Fbth6LezbnWuVQQYu5z+rAU7E= github.com/tliron/kutil v0.2.11/go.mod h1:sLKEUxnqQ3iks5qv0d8hNLmk5kIerpZ58fJALtwSn9M= github.com/tliron/yamlkeys v1.3.6 h1:PPV4q7flMqIvmSUSsEZuns7Qt3VIMxkhBj+6KTRvI9c= diff --git a/library/default.pgo b/library/default.pgo index 6e3304eb..536c313e 100644 Binary files a/library/default.pgo and b/library/default.pgo differ diff --git a/library/library.go b/library/library.go index 7e8c87b1..e804c88f 100644 --- a/library/library.go +++ b/library/library.go @@ -13,13 +13,13 @@ import ( "github.com/tliron/kutil/transcribe" cloutpkg "github.com/tliron/puccini/clout" "github.com/tliron/puccini/clout/js" - "github.com/tliron/puccini/tosca/parser" + "github.com/tliron/puccini/normal" + parserpkg "github.com/tliron/puccini/tosca/parser" "github.com/tliron/puccini/tosca/parsing" "github.com/tliron/yamlkeys" ) -import "github.com/tliron/puccini/normal" -var parser_ = parser.NewParser() +var parser = parserpkg.NewParser() //export Compile func Compile(url *C.char, inputs *C.char, quirks *C.char, resolve C.char, coerce C.char) *C.char { @@ -66,11 +66,11 @@ func Compile(url *C.char, inputs *C.char, quirks *C.char, resolve C.char, coerce var url_ exturl.URL var err error - if url_, err = urlContext.NewValidURL(context, C.GoString(url), nil); err != nil { + if url_, err = urlContext.NewValidAnyOrFileURL(context, C.GoString(url), nil); err != nil { return result(nil, nil, err) } - parserContext := parser_.NewContext() + parserContext := parser.NewContext() parserContext.URL = url_ parserContext.Quirks = quirks_ parserContext.Inputs = inputs_ diff --git a/puccini-clout/commands/common.go b/puccini-clout/commands/common.go index fcb7b65c..0b16500d 100644 --- a/puccini-clout/commands/common.go +++ b/puccini-clout/commands/common.go @@ -1,7 +1,13 @@ package commands import ( + contextpkg "context" + "github.com/tliron/commonlog" + "github.com/tliron/exturl" + "github.com/tliron/kutil/util" + "github.com/tliron/puccini/clout" + cloutpkg "github.com/tliron/puccini/clout" ) const toolName = "puccini-clout" @@ -9,3 +15,25 @@ const toolName = "puccini-clout" var log = commonlog.GetLogger(toolName) var output string + +func Bases(urlContext *exturl.Context) []exturl.URL { + workingDir, err := urlContext.NewWorkingDirFileURL() + util.FailOnError(err) + return []exturl.URL{workingDir} +} + +func LoadClout(context contextpkg.Context, url string, urlContext *exturl.Context) *clout.Clout { + var url_ exturl.URL + var err error + if url != "" { + url_, err = urlContext.NewValidAnyOrFileURL(context, url, Bases(urlContext)) + util.FailOnError(err) + } else { + url_, err = urlContext.ReadToInternalURLFromStdin(context, format) + util.FailOnError(err) + } + + clout, err := cloutpkg.Load(context, url_) + util.FailOnError(err) + return clout +} diff --git a/puccini-clout/commands/scriptlet-exec.go b/puccini-clout/commands/scriptlet-exec.go index 913d3fa2..f1461ec1 100644 --- a/puccini-clout/commands/scriptlet-exec.go +++ b/puccini-clout/commands/scriptlet-exec.go @@ -36,15 +36,14 @@ var execCommand = &cobra.Command{ defer urlContext.Release() context := contextpkg.TODO() - clout, err := cloutpkg.Load(context, url, inputFormat, urlContext) - util.FailOnError(err) + clout := LoadClout(context, url, urlContext) // Try loading JavaScript from Clout scriptlet, err := js.GetScriptlet(scriptletName, clout) if err != nil { // Try loading JavaScript from path or URL - scriptletUrl, err := urlContext.NewValidURL(context, scriptletName, nil) + scriptletUrl, err := urlContext.NewValidAnyOrFileURL(context, scriptletName, Bases(urlContext)) util.FailOnError(err) scriptlet, err = exturl.ReadString(context, scriptletUrl) @@ -62,5 +61,5 @@ var execCommand = &cobra.Command{ func Exec(scriptletName string, scriptlet string, clout *cloutpkg.Clout, urlContext *exturl.Context) error { jsContext := js.NewContext(scriptletName, log, arguments, terminal.Quiet, format, strict, pretty, output, urlContext) _, err := jsContext.Require(clout, scriptletName, nil) - return js.UnwrapException(err) + return err } diff --git a/puccini-clout/commands/scriptlet-get.go b/puccini-clout/commands/scriptlet-get.go index 36e77a7f..cea40f6f 100644 --- a/puccini-clout/commands/scriptlet-get.go +++ b/puccini-clout/commands/scriptlet-get.go @@ -9,7 +9,6 @@ import ( "github.com/tliron/kutil/terminal" "github.com/tliron/kutil/transcribe" "github.com/tliron/kutil/util" - cloutpkg "github.com/tliron/puccini/clout" "github.com/tliron/puccini/clout/js" ) @@ -34,8 +33,7 @@ var getCommand = &cobra.Command{ urlContext := exturl.NewContext() defer urlContext.Release() - clout, err := cloutpkg.Load(contextpkg.TODO(), url, inputFormat, urlContext) - util.FailOnError(err) + clout := LoadClout(contextpkg.TODO(), url, urlContext) scriptlet, err := js.GetScriptlet(scriptletName, clout) util.FailOnError(err) diff --git a/puccini-clout/commands/scriptlet-list.go b/puccini-clout/commands/scriptlet-list.go index 20366afc..025ebd05 100644 --- a/puccini-clout/commands/scriptlet-list.go +++ b/puccini-clout/commands/scriptlet-list.go @@ -1,7 +1,7 @@ package commands import ( - "context" + contextpkg "context" "strings" "github.com/spf13/cobra" @@ -31,8 +31,7 @@ var listCommand = &cobra.Command{ urlContext := exturl.NewContext() defer urlContext.Release() - clout, err := cloutpkg.Load(context.TODO(), url, inputFormat, urlContext) - util.FailOnError(err) + clout := LoadClout(contextpkg.TODO(), url, urlContext) List(clout) }, diff --git a/puccini-clout/commands/scriptlet-put.go b/puccini-clout/commands/scriptlet-put.go index 82a8cb28..26c75037 100644 --- a/puccini-clout/commands/scriptlet-put.go +++ b/puccini-clout/commands/scriptlet-put.go @@ -8,7 +8,6 @@ import ( "github.com/tliron/exturl" "github.com/tliron/kutil/transcribe" "github.com/tliron/kutil/util" - cloutpkg "github.com/tliron/puccini/clout" "github.com/tliron/puccini/clout/js" ) @@ -35,10 +34,9 @@ var putCommand = &cobra.Command{ defer urlContext.Release() context := contextpkg.TODO() - clout, err := cloutpkg.Load(context, url, inputFormat, urlContext) - util.FailOnError(err) + clout := LoadClout(context, url, urlContext) - scriptletUrl_, err := urlContext.NewValidURL(context, scriptletUrl, nil) + scriptletUrl_, err := urlContext.NewValidAnyOrFileURL(context, scriptletUrl, Bases(urlContext)) util.FailOnError(err) scriptlet, err := exturl.ReadString(context, scriptletUrl_) diff --git a/puccini-clout/default.pgo b/puccini-clout/default.pgo index 40df0cb1..21358112 100644 Binary files a/puccini-clout/default.pgo and b/puccini-clout/default.pgo differ diff --git a/puccini-csar/commands/common.go b/puccini-csar/commands/common.go index 80797cf0..d2f0dc8e 100644 --- a/puccini-csar/commands/common.go +++ b/puccini-csar/commands/common.go @@ -2,6 +2,8 @@ package commands import ( "github.com/tliron/commonlog" + "github.com/tliron/exturl" + "github.com/tliron/kutil/util" ) const toolName = "puccini-csar" @@ -9,3 +11,9 @@ const toolName = "puccini-csar" var log = commonlog.GetLogger(toolName) var archiveFormat string + +func Bases(urlContext *exturl.Context) []exturl.URL { + workingDir, err := urlContext.NewWorkingDirFileURL() + util.FailOnError(err) + return []exturl.URL{workingDir} +} diff --git a/puccini-csar/commands/create.go b/puccini-csar/commands/create.go index 1838d3e0..e01cc31b 100644 --- a/puccini-csar/commands/create.go +++ b/puccini-csar/commands/create.go @@ -51,13 +51,7 @@ var createCommand = &cobra.Command{ func CreateCSAR(csarPath string, dir string) { if (compressionLevel < 0) || (compressionLevel > 9) { - util.Failf("invalid compression level: %d", compressionLevel) - } - - stat, err := os.Stat(dir) - util.FailOnError(err) - if !stat.IsDir() { - util.Failf("not a directory: %s", dir) + util.Failf("invalid compression level, must be >=0 and <=9: %d", compressionLevel) } if archiveFormat == "" { @@ -68,147 +62,162 @@ func CreateCSAR(csarPath string, dir string) { util.Failf("unsupported CSAR archive format: %q", archiveFormat) } + stat, err := os.Stat(dir) + util.FailOnError(err) + if !stat.IsDir() { + util.Failf("not a directory: %s", dir) + } + file, err := os.OpenFile(csarPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) util.FailOnError(err) - write := func(w func(string, []byte, *os.File) error) { - prefix := len(dir) + 1 - var hasMeta bool - err = filepath.WalkDir(dir, func(path string, dirEntry fs.DirEntry, err error) error { - if err != nil { - return err - } + switch archiveFormat { + case "tar": + CreateTarCSAR(dir, file) - if !dirEntry.IsDir() { - internalPath := path[prefix:] - if internalPath == csar.TOSCA_META_PATH { - // Validate meta - _, err = csar.ReadMetaFromPath(path) - util.FailOnError(err) + case "tar.gz": + CreateGzipTarCSAR(dir, file) - hasMeta = true - log.Infof("using included %s", csar.TOSCA_META_PATH) - } - log.Infof("adding: %s", internalPath) - file, err := os.Open(path) - util.FailOnError(err) - defer file.Close() - return w(internalPath, nil, file) - } + case "zip", "csar": + CreateZipCSAR(dir, file) + } - return nil - }) - util.FailOnError(err) + util.OnExit(func() { + log.Noticef("created CSAR: %s", csarPath) + }) +} - if !hasMeta { - if entryDefinitions == "" { - dirEntries, err := os.ReadDir(dir) - util.FailOnError(err) - for _, dirEntry := range dirEntries { - name := dirEntry.Name() - if strings.HasSuffix(name, ".yaml") || strings.HasSuffix(name, ".yml") { - if entryDefinitions != "" { - util.Failf("dir has more than one potential service template at the root: %s", dir) - } else { - entryDefinitions = name - } - } - } - } +func CreateTarCSAR(dir string, writer io.Writer) { + tarWriter := tar.NewWriter(writer) + util.OnExitError(tarWriter.Close) - log.Infof("generating new %s", csar.TOSCA_META_PATH) + createCsar(dir, func(internalPath string, buffer []byte, file *os.File) error { + internalPath = filepath.ToSlash(internalPath) - toscaMetaFileVersion_, err := csar.ParseVersion(toscaMetaFileVersion) - util.FailOnError(err) - csarVersion_, err := csar.ParseVersion(csarVersion) - util.FailOnError(err) + header := tar.Header{ + Name: internalPath, + Mode: 0600, + } - meta := csar.Meta{ - Version: toscaMetaFileVersion_, - CsarVersion: csarVersion_, - CreatedBy: createdBy, - EntryDefinitions: entryDefinitions, - OtherDefinitions: otherDefinitions, + if buffer != nil { + header.Size = int64(len(buffer)) + } else { + if stat, err := file.Stat(); err == nil { + header.Size = stat.Size() + } else { + return err } + } - meta_, err := meta.ToBytes() - util.FailOnError(err) - - err = w(csar.TOSCA_META_PATH, meta_, nil) - util.FailOnError(err) + if err := tarWriter.WriteHeader(&header); err == nil { + if buffer != nil { + _, err = tarWriter.Write(buffer) + } else { + _, err = io.Copy(tarWriter, file) + } + return err + } else { + return err } - } + }) +} - writeTar := func(writer io.Writer) { - tarWriter := tar.NewWriter(writer) - util.OnExitError(tarWriter.Close) +func CreateGzipTarCSAR(dir string, writer io.Writer) { + gzipWriter, err := pgzip.NewWriterLevel(writer, compressionLevel) + util.FailOnError(err) + util.OnExitError(gzipWriter.Close) - write(func(internalPath string, buffer []byte, file *os.File) error { - internalPath = filepath.ToSlash(internalPath) + CreateTarCSAR(dir, gzipWriter) +} - header := tar.Header{ - Name: internalPath, - Mode: 0600, - } +func CreateZipCSAR(dir string, file *os.File) { + zipWriter := zip.NewWriter(file) + util.OnExitError(zipWriter.Close) + zipWriter.RegisterCompressor(zip.Deflate, func(out io.Writer) (io.WriteCloser, error) { + return flate.NewWriter(out, compressionLevel) + }) + + createCsar(dir, func(internalPath string, buffer []byte, file *os.File) error { + internalPath = filepath.ToSlash(internalPath) + if writer, err := zipWriter.Create(internalPath); err == nil { if buffer != nil { - header.Size = int64(len(buffer)) + _, err = writer.Write(buffer) } else { - if stat, err := file.Stat(); err == nil { - header.Size = stat.Size() - } else { - return err - } + _, err = io.Copy(writer, file) } + return err + } else { + return err + } + }) +} + +func createCsar(dir string, writeEntry func(string, []byte, *os.File) error) { + prefix := len(dir) + 1 + var hasMeta bool + + err := filepath.WalkDir(dir, func(path string, dirEntry fs.DirEntry, err error) error { + if err != nil { + return err + } - if err = tarWriter.WriteHeader(&header); err == nil { - if buffer != nil { - _, err = tarWriter.Write(buffer) - } else { - _, err = io.Copy(tarWriter, file) + if !dirEntry.IsDir() { + internalPath := path[prefix:] + if internalPath == csar.TOSCA_META_PATH { + // Validate meta + _, err = csar.ReadMetaFromPath(path) + util.FailOnError(err) + + hasMeta = true + log.Infof("using included %s", csar.TOSCA_META_PATH) + } + log.Infof("adding: %s", internalPath) + file, err := os.Open(path) + util.FailOnError(err) + defer file.Close() + return writeEntry(internalPath, nil, file) + } + + return nil + }) + util.FailOnError(err) + + if !hasMeta { + if entryDefinitions == "" { + dirEntries, err := os.ReadDir(dir) + util.FailOnError(err) + for _, dirEntry := range dirEntries { + name := dirEntry.Name() + if strings.HasSuffix(name, ".yaml") || strings.HasSuffix(name, ".yml") { + if entryDefinitions != "" { + util.Failf("dir has more than one potential service template at the root: %s", dir) + } else { + entryDefinitions = name + } } - return err - } else { - return err } - }) - } + } - switch archiveFormat { - case "tar": - writeTar(file) + log.Infof("generating new %s", csar.TOSCA_META_PATH) - case "tar.gz": - gzipWriter, err := pgzip.NewWriterLevel(file, compressionLevel) + toscaMetaFileVersion_, err := csar.ParseVersion(toscaMetaFileVersion) + util.FailOnError(err) + csarVersion_, err := csar.ParseVersion(csarVersion) util.FailOnError(err) - util.OnExitError(gzipWriter.Close) - writeTar(gzipWriter) + meta := csar.Meta{ + Version: toscaMetaFileVersion_, + CsarVersion: csarVersion_, + CreatedBy: createdBy, + EntryDefinitions: entryDefinitions, + OtherDefinitions: otherDefinitions, + } - case "zip", "csar": - zipWriter := zip.NewWriter(file) - util.OnExitError(zipWriter.Close) - - zipWriter.RegisterCompressor(zip.Deflate, func(out io.Writer) (io.WriteCloser, error) { - return flate.NewWriter(out, compressionLevel) - }) - - write(func(internalPath string, buffer []byte, file *os.File) error { - internalPath = filepath.ToSlash(internalPath) - if writer, err := zipWriter.Create(internalPath); err == nil { - if buffer != nil { - _, err = writer.Write(buffer) - } else { - _, err = io.Copy(writer, file) - } - return err - } else { - return err - } - }) - } + meta_, err := meta.ToBytes() + util.FailOnError(err) - util.OnExit(func() { - log.Noticef("created CSAR: %s", csarPath) - }) + err = writeEntry(csar.TOSCA_META_PATH, meta_, nil) + util.FailOnError(err) + } } diff --git a/puccini-csar/commands/meta.go b/puccini-csar/commands/meta.go index deaebe5b..42b0f1e8 100644 --- a/puccini-csar/commands/meta.go +++ b/puccini-csar/commands/meta.go @@ -49,7 +49,7 @@ func Meta(url string) { context := contextpkg.TODO() var csarUrl exturl.URL - csarUrl, err = urlContext.NewValidURL(context, url, nil) + csarUrl, err = urlContext.NewValidAnyOrFileURL(context, url, Bases(urlContext)) util.FailOnError(err) var meta *csar.Meta diff --git a/puccini-csar/default.pgo b/puccini-csar/default.pgo index 2021208d..12ddc4e5 100644 Binary files a/puccini-csar/default.pgo and b/puccini-csar/default.pgo differ diff --git a/puccini-tosca/commands/common.go b/puccini-tosca/commands/common.go index d10afc8c..850f9768 100644 --- a/puccini-tosca/commands/common.go +++ b/puccini-tosca/commands/common.go @@ -4,6 +4,7 @@ import ( "os" "github.com/tliron/commonlog" + "github.com/tliron/exturl" problemspkg "github.com/tliron/kutil/problems" "github.com/tliron/kutil/terminal" "github.com/tliron/kutil/transcribe" @@ -35,3 +36,19 @@ func FailOnProblems(problems *problemspkg.Problems) { util.Exit(1) } } + +func Bases(urlContext *exturl.Context, withImportPaths bool) []exturl.URL { + var bases []exturl.URL + + if withImportPaths { + for _, importPath := range importPaths { + bases = append(bases, urlContext.NewAnyOrFileURL(importPath)) + } + } + + workingDir, err := urlContext.NewWorkingDirFileURL() + util.FailOnError(err) + bases = append(bases, workingDir) + + return bases +} diff --git a/puccini-tosca/commands/compile.go b/puccini-tosca/commands/compile.go index 0b54abad..3c9dcc11 100644 --- a/puccini-tosca/commands/compile.go +++ b/puccini-tosca/commands/compile.go @@ -99,7 +99,7 @@ func Exec(context contextpkg.Context, scriptletName string, arguments map[string if err != nil { // Try loading JavaScript from path or URL - url, err := urlContext.NewValidURL(context, scriptletName, nil) + url, err := urlContext.NewValidAnyOrFileURL(context, scriptletName, Bases(urlContext, false)) util.FailOnError(err) scriptlet, err = exturl.ReadString(context, url) @@ -111,5 +111,5 @@ func Exec(context contextpkg.Context, scriptletName string, arguments map[string jsContext := js.NewContext(scriptletName, log, arguments, terminal.Quiet, format, strict, pretty, output, urlContext) _, err = jsContext.Require(clout, scriptletName, nil) - return js.UnwrapException(err) + return err } diff --git a/puccini-tosca/commands/parse.go b/puccini-tosca/commands/parse.go index 437ed25c..0ea851a2 100644 --- a/puccini-tosca/commands/parse.go +++ b/puccini-tosca/commands/parse.go @@ -12,7 +12,7 @@ import ( "github.com/tliron/kutil/transcribe" "github.com/tliron/kutil/util" "github.com/tliron/puccini/normal" - "github.com/tliron/puccini/tosca/parser" + parserpkg "github.com/tliron/puccini/tosca/parser" "github.com/tliron/puccini/tosca/parsing" "github.com/tliron/yamlkeys" ) @@ -66,9 +66,9 @@ var parseCommand = &cobra.Command{ }, } -var parser_ = parser.NewParser() +var parser = parserpkg.NewParser() -func Parse(context contextpkg.Context, url string) (*parser.Context, *normal.ServiceTemplate) { +func Parse(context contextpkg.Context, url string) (*parserpkg.Context, *normal.ServiceTemplate) { ParseInputs(context) urlContext := exturl.NewContext() @@ -79,15 +79,6 @@ func Parse(context contextpkg.Context, url string) (*parser.Context, *normal.Ser urlContext.Map(fromUrl, toUrl) } - var origins []exturl.URL - for _, importPath := range importPaths { - origin, err := urlContext.NewURL(importPath) - if err != nil { - origin = urlContext.NewFileURL(importPath) - } - origins = append(origins, origin) - } - var url_ exturl.URL var err error if url == "" { @@ -95,11 +86,11 @@ func Parse(context contextpkg.Context, url string) (*parser.Context, *normal.Ser url_, err = urlContext.ReadToInternalURLFromStdin(context, "yaml") } else { log.Infof("parsing %q", url) - url_, err = urlContext.NewValidURL(context, url, origins) + url_, err = urlContext.NewValidAnyOrFileURL(context, url, Bases(urlContext, false)) } util.FailOnError(err) - parserContext := parser_.NewContext() + parserContext := parser.NewContext() parserContext.Quirks = parsing.NewQuirks(quirks...) parserContext.Stylist = terminal.DefaultStylist if problemsFormat != "" { @@ -111,7 +102,7 @@ func Parse(context contextpkg.Context, url string) (*parser.Context, *normal.Ser } // Phase 1: Read - ok := parserContext.ReadRoot(context, url_, origins, template) + ok := parserContext.ReadRoot(context, url_, Bases(urlContext, true), template) parserContext.MergeProblems() problems := parserContext.GetProblems() @@ -168,7 +159,7 @@ func Parse(context contextpkg.Context, url string) (*parser.Context, *normal.Ser // Phase 4: Inheritance if ToPrintPhase(4) { - parserContext.Inherit(func(tasks parser.Tasks) { + parserContext.Inherit(func(tasks parserpkg.Tasks) { if len(dumpPhases) > 1 { terminal.Printf("%s\n", terminal.DefaultStylist.Heading("Inheritance Tasks")) tasks.Print(1) @@ -259,7 +250,7 @@ func ParseInputs(context contextpkg.Context) { urlContext := exturl.NewContext() util.OnExitError(urlContext.Release) - url, err := urlContext.NewValidURL(context, inputsUrl, nil) + url, err := urlContext.NewValidAnyOrFileURL(context, inputsUrl, Bases(urlContext, false)) util.FailOnError(err) reader, err := url.Open(context) util.FailOnError(err) @@ -278,11 +269,9 @@ func ParseInputs(context contextpkg.Context) { } } - if inputs != nil { - for name, input := range inputs { - input_, _, err := ard.DecodeYAML(input, false) - util.FailOnError(err) - inputValues[name] = input_ - } + for name, input := range inputs { + input_, _, err := ard.DecodeYAML(input, false) + util.FailOnError(err) + inputValues[name] = input_ } } diff --git a/puccini-tosca/default.pgo b/puccini-tosca/default.pgo index 6e3304eb..536c313e 100644 Binary files a/puccini-tosca/default.pgo and b/puccini-tosca/default.pgo differ diff --git a/tosca/csar/paths.go b/tosca/csar/paths.go index 4cf6df87..7941db51 100644 --- a/tosca/csar/paths.go +++ b/tosca/csar/paths.go @@ -8,7 +8,6 @@ import ( "strings" "github.com/klauspost/compress/zip" - "github.com/tliron/exturl" ) diff --git a/tosca/grammars/cloudify_v1_3/import.go b/tosca/grammars/cloudify_v1_3/import.go index d5069c2d..4c5680fe 100644 --- a/tosca/grammars/cloudify_v1_3/import.go +++ b/tosca/grammars/cloudify_v1_3/import.go @@ -52,9 +52,9 @@ func (self *Import) NewImportSpec(unit *File) (*parsing.ImportSpec, bool) { file = s[1] } - origin := self.Context.URL.Origin() - var origins = []exturl.URL{origin} - url, err := origin.Context().NewValidURL(contextpkg.TODO(), file, origins) + base := self.Context.URL.Base() + var bases = []exturl.URL{base} + url, err := base.Context().NewValidAnyOrFileURL(contextpkg.TODO(), file, bases) if err != nil { self.Context.ReportError(err) return nil, false diff --git a/tosca/grammars/tosca_v2_0/artifact-definition.go b/tosca/grammars/tosca_v2_0/artifact-definition.go index be6a805b..1a84aca5 100644 --- a/tosca/grammars/tosca_v2_0/artifact-definition.go +++ b/tosca/grammars/tosca_v2_0/artifact-definition.go @@ -79,10 +79,10 @@ func (self *ArtifactDefinition) GetURL(context contextpkg.Context) exturl.URL { self.url = url.Relative(*self.File) } } else { - origin := self.Context.URL.Origin() - origins := []exturl.URL{origin} + base := self.Context.URL.Base() + bases := []exturl.URL{base} var err error - if self.url, err = origin.Context().NewValidURL(context, *self.File, origins); err != nil { + if self.url, err = base.Context().NewValidAnyOrFileURL(context, *self.File, bases); err != nil { // Avoid reporting more than once if !self.urlProblemReported { self.Context.ReportError(err) diff --git a/tosca/grammars/tosca_v2_0/import.go b/tosca/grammars/tosca_v2_0/import.go index 779c0a25..a9c26396 100644 --- a/tosca/grammars/tosca_v2_0/import.go +++ b/tosca/grammars/tosca_v2_0/import.go @@ -85,7 +85,7 @@ func (self *Import) NewImportSpec(unit *File) (*parsing.ImportSpec, bool) { } } - var origins []exturl.URL + var bases []exturl.URL var urlContext *exturl.Context if repository != nil { @@ -95,17 +95,17 @@ func (self *Import) NewImportSpec(unit *File) (*parsing.ImportSpec, bool) { return nil, false } - origins = []exturl.URL{repositoryUrl} + bases = []exturl.URL{repositoryUrl} urlContext = repositoryUrl.Context() } else { - origin := self.Context.URL.Origin() - origins = []exturl.URL{origin} - urlContext = origin.Context() + base := self.Context.URL.Base() + bases = []exturl.URL{base} + urlContext = base.Context() } - origins = append(origins, self.Context.Origins...) + bases = append(bases, self.Context.Bases...) - url, err := urlContext.NewValidURL(context.TODO(), *self.URL, origins) + url, err := urlContext.NewValidAnyOrFileURL(context.TODO(), *self.URL, bases) if err != nil { self.Context.ReportError(err) return nil, false diff --git a/tosca/grammars/tosca_v2_0/repository.go b/tosca/grammars/tosca_v2_0/repository.go index 3865fdd9..69e859b3 100644 --- a/tosca/grammars/tosca_v2_0/repository.go +++ b/tosca/grammars/tosca_v2_0/repository.go @@ -24,8 +24,7 @@ type Repository struct { URL *string `read:"url" mandatory:""` Credential *Value `read:"credential,Value"` // tosca:Credential - url exturl.URL - urlProblemReported bool + url exturl.URL } func NewRepository(context *parsing.Context) *Repository { @@ -64,15 +63,7 @@ func (self *Repository) render() { func (self *Repository) GetURL() exturl.URL { if (self.url == nil) && (self.URL != nil) { - origin := self.Context.URL.Origin() - var err error - if self.url, err = origin.Context().NewURL(*self.URL); err != nil { - // Avoid reporting more than once - if !self.urlProblemReported { - self.Context.ReportError(err) - self.urlProblemReported = true - } - } + self.url = self.Context.URL.Context().NewAnyOrFileURL(*self.URL) } return nil diff --git a/tosca/parser/context.go b/tosca/parser/context.go index 37981b4a..483933f3 100644 --- a/tosca/parser/context.go +++ b/tosca/parser/context.go @@ -21,7 +21,7 @@ type Context struct { Parser *Parser URL exturl.URL - Origins []exturl.URL + Bases []exturl.URL Quirks parsing.Quirks Inputs map[string]ard.Value Stylist *terminal.Stylist diff --git a/tosca/parser/parse.go b/tosca/parser/parse.go index 4e0a77fe..410c8286 100644 --- a/tosca/parser/parse.go +++ b/tosca/parser/parse.go @@ -9,7 +9,7 @@ import ( func (self *Context) Parse(context contextpkg.Context) (*normal.ServiceTemplate, error) { // Phase 1: Read - ok := self.ReadRoot(context, self.URL, self.Origins, "") + ok := self.ReadRoot(context, self.URL, self.Bases, "") self.MergeProblems() problems := self.GetProblems() diff --git a/tosca/parser/phase1-read.go b/tosca/parser/phase1-read.go index b46859f8..10f93c28 100644 --- a/tosca/parser/phase1-read.go +++ b/tosca/parser/phase1-read.go @@ -15,9 +15,9 @@ import ( "github.com/tliron/yamlkeys" ) -func (self *Context) ReadRoot(context contextpkg.Context, url exturl.URL, origins []exturl.URL, template string) bool { +func (self *Context) ReadRoot(context contextpkg.Context, url exturl.URL, bases []exturl.URL, template string) bool { parsingContext := parsing.NewContext(self.Stylist, self.Quirks) - parsingContext.Origins = origins + parsingContext.Bases = bases parsingContext.URL = url diff --git a/tosca/parsing/context.go b/tosca/parsing/context.go index 8627aee4..2267b8c8 100644 --- a/tosca/parsing/context.go +++ b/tosca/parsing/context.go @@ -55,7 +55,7 @@ type Context struct { Name string Path ard.Path URL exturl.URL - Origins []exturl.URL + Bases []exturl.URL Data ard.Value Locator ard.Locator CanonicalNamespace *string @@ -88,7 +88,7 @@ func (self *Context) NewImportContext(url exturl.URL) *Context { Name: self.Name, Path: self.Path, URL: url, - Origins: self.Origins, + Bases: self.Bases, CanonicalNamespace: self.CanonicalNamespace, Namespace: NewNamespace(), ScriptletNamespace: NewScriptletNamespace(), @@ -164,7 +164,7 @@ func (self *Context) Clone(data ard.Value) *Context { Name: self.Name, Path: self.Path, URL: self.URL, - Origins: self.Origins, + Bases: self.Bases, Data: data, Locator: self.Locator, CanonicalNamespace: self.CanonicalNamespace, @@ -185,7 +185,7 @@ func (self *Context) FieldChild(name ard.Value, data ard.Value) *Context { Name: nameString, Path: self.Path.AppendField(nameString), URL: self.URL, - Origins: self.Origins, + Bases: self.Bases, Data: data, Locator: self.Locator, CanonicalNamespace: self.CanonicalNamespace, @@ -232,7 +232,7 @@ func (self *Context) MapChild(name ard.Value, data ard.Value) *Context { Name: nameString, Path: self.Path.AppendMap(nameString), URL: self.URL, - Origins: self.Origins, + Bases: self.Bases, Data: data, Locator: self.Locator, CanonicalNamespace: self.CanonicalNamespace, @@ -252,7 +252,7 @@ func (self *Context) ListChild(index int, data ard.Value) *Context { Name: strconv.FormatInt(int64(index), 10), Path: self.Path.AppendList(index), URL: self.URL, - Origins: self.Origins, + Bases: self.Bases, Data: data, Locator: self.Locator, CanonicalNamespace: self.CanonicalNamespace, @@ -272,7 +272,7 @@ func (self *Context) SequencedListChild(index int, name string, data ard.Value) Name: name, Path: self.Path.AppendSequencedList(index), URL: self.URL, - Origins: self.Origins, + Bases: self.Bases, Data: data, Locator: self.Locator, CanonicalNamespace: self.CanonicalNamespace, diff --git a/tosca/parsing/scriptlets.go b/tosca/parsing/scriptlets.go index c70034dc..497cfea3 100644 --- a/tosca/parsing/scriptlets.go +++ b/tosca/parsing/scriptlets.go @@ -13,7 +13,7 @@ func (self *Context) ImportScriptlet(name string, path string) { var nativeArgumentIndexes []int name, nativeArgumentIndexes = parseScriptletName(name) self.ScriptletNamespace.Set(name, &Scriptlet{ - Origin: self.URL.Origin(), + Base: self.URL.Base(), Path: path, NativeArgumentIndexes: nativeArgumentIndexes, }) @@ -30,7 +30,7 @@ func (self *Context) EmbedScriptlet(name string, scriptlet string) { // type Scriptlet struct { - Origin exturl.URL `json:"origin" yaml:"origin"` + Base exturl.URL `json:"base" yaml:"base"` Path string `json:"path" yaml:"path"` Scriptlet string `json:"scriptlet" yaml:"scriptlet"` NativeArgumentIndexes []int `json:"nativeArgumentIndexes" yaml:"nativeArgumentIndexes"` @@ -38,17 +38,17 @@ type Scriptlet struct { func (self *Scriptlet) Read(context contextpkg.Context) (string, error) { if self.Path != "" { - var origins []exturl.URL + var bases []exturl.URL var urlContext *exturl.Context - if self.Origin != nil { - origins = []exturl.URL{self.Origin} - urlContext = self.Origin.Context() + if self.Base != nil { + bases = []exturl.URL{self.Base} + urlContext = self.Base.Context() } else { urlContext = exturl.NewContext() defer urlContext.Release() } - url, err := urlContext.NewValidURL(context, self.Path, origins) + url, err := urlContext.NewValidAnyOrFileURL(context, self.Path, bases) if err != nil { return "", err }