diff --git a/models/examples/if-else-statement.json b/models/examples/if-else-statement.json index 2d17e4a..3f02ee4 100644 --- a/models/examples/if-else-statement.json +++ b/models/examples/if-else-statement.json @@ -5,6 +5,14 @@ { "value": "10", "target": "max" + }, + { + "value": "SECRET_PASS", + "target": "secretL1" + }, + { + "value": "SECRET_PASS", + "target": "secretL2" } ], "workflow": @@ -18,7 +26,9 @@ "uid": "192161d7-e3f2-4991-adc0-a99c88c144c0", "description": "Graph component", "inputs": [ - { "name": "max", "mediatype": ["integer"], "type": "parameter" } + { "name": "max", "mediatype": ["integer"], "type": "parameter" }, + { "name": "secretL1", "type": "env_secret" }, + { "name": "secretL2", "type": "env_secret" } ], "outputs": [ { "name": "description", "type": "parameter" } @@ -30,6 +40,14 @@ { "source": { "port": "max" }, "target": { "node": "N1", "port": "value" } + }, + { + "source": { "port": "secretL1" }, + "target": { "node": "N1", "port": "secretB1" } + }, + { + "source": { "port": "secretL2" }, + "target": { "node": "If", "port": "secretPass" } } ], "outputMappings": [ @@ -46,7 +64,8 @@ "uid": "192161d7-e3f2-4991-adc0-a99c88c144b0", "description": "Generate", "inputs": [ - { "name": "value", "type": "parameter" } + { "name": "value", "type": "parameter" }, + { "name": "secretB1", "type": "env_secret" } ], "outputs": [ { "name": "rand", "type": "parameter" } @@ -57,7 +76,7 @@ "container": { "name": "containername_n1_b1", "image": "bash:latest", - "command": ["bash", "-c", "shuf -i 0-$0 -n1 > /tmp/out"], + "command": ["bash", "-c", "echo SecretB1 $secretB1; shuf -i 0-$0 -n1 > /tmp/out"], "args": [] }, "args": [ @@ -85,7 +104,8 @@ "name": "valFromParam", "mediatype": ["number"], "type": "parameter" - } + }, + { "name": "secretPass", "type": "env_secret" } ], "outputs": [ { "name": "ifOut", "type": "parameter" } @@ -97,6 +117,10 @@ { "source": { "port": "valFromParam" }, "target": { "port": "valParam" } + }, + { + "source": { "port": "secretPass" }, + "target": { "port": "envValue" } } ], "outputMappings": [ @@ -113,6 +137,10 @@ "name": "valParam", "mediatype": ["number"], "type": "parameter" + }, + { + "name": "envValue", + "type": "env_secret" } ], "outputs": [ @@ -127,7 +155,7 @@ "command": [ "sh", "-c", - "echo value $0 is huge > /tmp/out" + "echo env $envValue; echo value $0 is huge > /tmp/out" ], "args": [] }, @@ -153,6 +181,10 @@ "name": "valParam", "mediatype": ["number"], "type": "parameter" + }, + { + "name": "envValue", + "type": "env_secret" } ], "outputs": [ @@ -167,7 +199,7 @@ "command": [ "sh", "-c", - "echo value $0 is small > /tmp/out" + "echo env $envValue; echo value $0 is small > /tmp/out" ], "args": [] }, diff --git a/transpiler/helpers.go b/transpiler/helpers.go index e28c2d2..ed507d8 100644 --- a/transpiler/helpers.go +++ b/transpiler/helpers.go @@ -52,8 +52,7 @@ func getNodeSecretMap(targetNodeId string, cmpSecrets secretMap, cmpInputs []mod for _, m := range inputsMap { mt, ok := checkInputType(cmpInputs, m.Source.Port) if ok && mt == models.FlowifySecretType && m.Target.Node == targetNodeId { - k, _ := findKeyFor(nSecrets, m.Source.Port) - nSecrets[k] = m.Target.Port + nSecrets[m.Target.Port] = cmpSecrets[m.Source.Port] } } return nSecrets diff --git a/transpiler/transpiler.go b/transpiler/transpiler.go index 52929a5..e0e27c7 100644 --- a/transpiler/transpiler.go +++ b/transpiler/transpiler.go @@ -72,9 +72,9 @@ func AddBrick(name string, cmp *models.Brick, outputs wfv1.Outputs, inputs wfv1. container.Args = args envs := []corev1.EnvVar{} for k, e := range sMap { - s := corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: secret.DefaultObjectName}, Key: k} + s := corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: secret.DefaultObjectName}, Key: e} vf := corev1.EnvVarSource{SecretKeyRef: &s} - env := corev1.EnvVar{Name: e, ValueFrom: &vf} + env := corev1.EnvVar{Name: k, ValueFrom: &vf} envs = append(envs, env) } container.Env = envs @@ -155,7 +155,7 @@ func AddGraph(name string, cmp *models.Graph, outputs wfv1.Outputs, inputs wfv1. errors.Errorf("Unrecognized implementation type: %s", impCmp) } case models.FlowifyVolumeType: - logrus.Info("Ref volume edge: ", e) + logrus.Debug("Ref volume edge:", e) default: return errors.Errorf("Unrecognized edge target-type, '%s', for target port-address '%s.%s'", it, e.Target.Node, e.Target.Port) } @@ -425,12 +425,11 @@ func TraverseComponent(cmp *models.Component, templates *[]wfv1.Template, tasks } inParams = append(inParams, param) case models.FlowifySecretType: - _, ok := findKeyFor(cmpSecrets, i.Name) - if !ok { - cmpSecrets[i.Name] = i.Name + if _, ok := cmpSecrets[i.Name]; !ok { + return nil, fmt.Errorf("missing secret <%s> for component: %s", i.Name, cmp.Uid) } case models.FlowifyVolumeType: - logrus.Info("Ref mount: ", i.Name, volumes[i.Name]) + logrus.Debugf("Ref mount: %s %s", i.Name, volumes[i.Name]) default: return nil, fmt.Errorf("cannot append input data (name: %s, type %s) at node %s", i.Name, i.Type, cmpName) } @@ -468,8 +467,7 @@ func TraverseComponent(cmp *models.Component, templates *[]wfv1.Template, tasks if mt, ok := checkInputType(cmp.Inputs, m.Source.Port); ok { switch mt { case models.FlowifySecretType: - k, _ := findKeyFor(nodeSecrets, m.Source.Port) - nodeSecrets[k] = m.Target.Port + nodeSecrets[m.Target.Port] = cmpSecrets[m.Source.Port] case models.FlowifyVolumeType: if val, ok := volumes[m.Source.Port]; ok { nodeVolumes[m.Target.Port] = val @@ -508,12 +506,11 @@ func TraverseComponent(cmp *models.Component, templates *[]wfv1.Template, tasks if mt, ok := checkInputType(cmp.Inputs, m.Source.Port); ok { switch mt { case models.FlowifySecretType: - k, _ := findKeyFor(nodeSecrets, m.Source.Port) - nodeSecrets[k] = m.Target.Port + nodeSecrets[m.Target.Port] = cmpSecrets[m.Source.Port] case models.FlowifyVolumeType: if val, ok := volumes[m.Source.Port]; ok { nodeVolumes[m.Target.Port] = val - logrus.Infof("Rewriting nodeVolumes for %s: %s -> %s", cmp.Name, m.Source.Port, m.Target.Port) + logrus.Debugf("Rewriting nodeVolumes for %s: %s -> %s", cmp.Name, m.Source.Port, m.Target.Port) } } } @@ -538,7 +535,7 @@ func TraverseComponent(cmp *models.Component, templates *[]wfv1.Template, tasks scopeVolumes[m.Target.Node] = make(map[string]corev1.Volume, 0) } scopeVolumes[m.Target.Node][m.Target.Port] = val - logrus.Infof("Rewriting nodeVolumes for %s: %s -> %s", cmp.Name, m.Source.Port, m.Target.Port) + logrus.Debugf("Rewriting nodeVolumes for %s: %s -> %s", cmp.Name, m.Source.Port, m.Target.Port) } } } @@ -551,17 +548,17 @@ func TraverseComponent(cmp *models.Component, templates *[]wfv1.Template, tasks } nodeSecrets := getNodeSecretMap(c.Id, cmpSecrets, cmp.Inputs, impCmp.InputMappings) volumes := getConnectedVolumeMap(c.Id, &tc, impCmp.Edges, impCmp.Nodes, scopeVolumes) - logrus.Info(volumes) + logrus.Debug(volumes) _, err := TraverseComponent(&tc, templates, tasks, nodeSecrets, volumes) if err != nil { return nil, err } } case models.Brick: - for k, elem := range cmpSecrets { + for k, _ := range cmpSecrets { ex := false for _, i := range cmp.Inputs { - if elem == i.Name { + if k == i.Name { ex = true break } @@ -605,15 +602,20 @@ func GetArgoWorkflow(job models.Job) (*wfv1.Workflow, error) { secretMapValues := make(secretMap) for _, cmpInput := range wf.Component.Inputs { if cmpInput.Type == models.FlowifySecretType { + missing := true for _, jobInput := range job.InputValues { if cmpInput.Name == jobInput.Target { val, ok := jobInput.Value.(string) if !ok { return nil, errors.Errorf("Cannot convert flowify secret '%s' to string.", jobInput.Target) } - secretMapValues[val] = cmpInput.Name + secretMapValues[cmpInput.Name] = val + missing = false } } + if missing { + return nil, errors.Errorf("Missing input secret <%s> for job: %s", cmpInput.Name, job.Name) + } } } @@ -637,7 +639,7 @@ func GetArgoWorkflow(job models.Job) (*wfv1.Workflow, error) { if err != nil { return nil, err } - logrus.Infof("Appending volume from config: %s -> (%s) ", config, v.Target) + logrus.Debugf("Appending volume from config: %s -> (%s) ", config, v.Target) volumeMap[tgt.Name] = volume // add volume with top level input name } else { return nil, fmt.Errorf("mount config must be json encoded string") diff --git a/transpiler/transpiler_test.go b/transpiler/transpiler_test.go index d76d875..c6912ad 100644 --- a/transpiler/transpiler_test.go +++ b/transpiler/transpiler_test.go @@ -187,6 +187,8 @@ const ( }` ) +var secrets = secretMap{"secretWF1": "wf1_secret_value", "secretWF2": "wf2_secret_value"} + func init() { } @@ -202,7 +204,7 @@ func Test_ParseComponentTree(t *testing.T) { assert.Equal(t, 3, len(wf.Component.Inputs)) assert.Equal(t, 0, len(wf.Component.Outputs)) - argoWF, err := ParseComponentTree(wf, secretMap{}, volumeMap{}, map[string]string{}, map[string]string{}) + argoWF, err := ParseComponentTree(wf, secrets, volumeMap{}, map[string]string{}, map[string]string{}) assert.Nil(t, err) assert.True(t, len(argoWF.Spec.Templates) == 4, "Expected no. of templates is 4.") // assert.Equal(t, "seedWF", argoWF.Spec.Arguments.Parameters[0].Name) @@ -222,9 +224,9 @@ func Test_ParseComponentTree(t *testing.T) { for _, env := range template.Container.Env { switch env.Name { case "secretN1": - assert.Equal(t, "secretWF1", env.ValueFrom.SecretKeyRef.Key) + assert.Equal(t, secrets["secretWF1"], env.ValueFrom.SecretKeyRef.Key) case "secretN2": - assert.Equal(t, "secretWF2", env.ValueFrom.SecretKeyRef.Key) + assert.Equal(t, secrets["secretWF2"], env.ValueFrom.SecretKeyRef.Key) default: t.Errorf("Unexpected secret %s in brick %s.", env.Name, template.Name) } @@ -232,7 +234,7 @@ func Test_ParseComponentTree(t *testing.T) { case "192161d7-e3f2-4991-adc0-a99c88c144b2": assert.Equal(t, 1, len(template.Container.Env)) assert.Equal(t, "secretN1", template.Container.Env[0].Name) - assert.Equal(t, "secretWF2", template.Container.Env[0].ValueFrom.SecretKeyRef.Key) + assert.Equal(t, secrets["secretWF2"], template.Container.Env[0].ValueFrom.SecretKeyRef.Key) default: t.Errorf("Unexpected template %s.", template.Name) } @@ -244,7 +246,8 @@ func Test_RemoveDuplicatedTemplates(t *testing.T) { err := json.Unmarshal([]byte(minimalExampleJSON), &wf) assert.Nil(t, err) - job := models.Job{Metadata: models.Metadata{Description: "test job"}, Type: "job", InputValues: nil, Workflow: wf} + inputValues := []models.Value{{Value: secrets["secretWF1"], Target: "secretWF1"}, {Value: secrets["secretWF2"], Target: "secretWF2"}} + job := models.Job{Metadata: models.Metadata{Description: "test job"}, Type: "job", InputValues: inputValues, Workflow: wf} argoWF, err := GetArgoWorkflow(job) assert.Nil(t, err) assert.True(t, len(argoWF.Spec.Templates) == 3, "Expected no. of templates is 3.")