Allowed multiple values of secrets (#34)

models/examples/if-else-statement.json

@@ -5,6 +5,14 @@
         {
             "value": "10",
             "target": "max"
+        },
+        {
+          "value": "SECRET_PASS",
+          "target": "secretL1"
+        },
+        {
+          "value": "SECRET_PASS",
+          "target": "secretL2"
         }
     ],
     "workflow":
@@ -18,7 +26,9 @@
             "uid": "192161d7-e3f2-4991-adc0-a99c88c144c0",
             "description": "Graph component",
             "inputs": [
-                { "name": "max", "mediatype": ["integer"], "type": "parameter" }
+                { "name": "max", "mediatype": ["integer"], "type": "parameter" },
+                { "name": "secretL1", "type": "env_secret" },
+                { "name": "secretL2", "type": "env_secret" }
             ],
             "outputs": [
                 { "name": "description", "type": "parameter" }
@@ -30,6 +40,14 @@
                     {
                         "source": { "port": "max" },
                         "target": { "node": "N1", "port": "value" }
+                    },
+                    {
+                      "source": { "port": "secretL1" },
+                      "target": { "node": "N1", "port": "secretB1" }
+                    },
+                    {
+                      "source": { "port": "secretL2" },
+                      "target": { "node": "If", "port": "secretPass" }
                     }
                 ],
                 "outputMappings": [
@@ -46,7 +64,8 @@
                             "uid": "192161d7-e3f2-4991-adc0-a99c88c144b0",
                             "description": "Generate",
                             "inputs": [
-                                { "name": "value", "type": "parameter" }
+                                { "name": "value", "type": "parameter" },
+                                { "name": "secretB1", "type": "env_secret" }
                             ],
                             "outputs": [
                                 { "name": "rand", "type": "parameter" }
@@ -57,7 +76,7 @@
                                 "container": {
                                     "name": "containername_n1_b1",
                                     "image": "bash:latest",
-                                    "command": ["bash", "-c", "shuf -i 0-$0 -n1 > /tmp/out"],
+                                    "command": ["bash", "-c", "echo SecretB1 $secretB1; shuf -i 0-$0 -n1 > /tmp/out"],
                                     "args": []
                                 },
                                 "args": [
@@ -85,7 +104,8 @@
                                     "name": "valFromParam",
                                     "mediatype": ["number"],
                                     "type": "parameter"
-                                }
+                                },
+                                { "name": "secretPass", "type": "env_secret" }
                             ],
                             "outputs": [
                                 { "name": "ifOut", "type": "parameter" }
@@ -97,6 +117,10 @@
                                     {
                                         "source": { "port": "valFromParam" },
                                         "target": { "port": "valParam" }
+                                    },
+                                    {
+                                        "source": { "port": "secretPass" },
+                                        "target": { "port": "envValue" }
                                     }
                                 ],
                                 "outputMappings": [
@@ -113,6 +137,10 @@
                                             "name": "valParam",
                                             "mediatype": ["number"],
                                             "type": "parameter"
+                                        },
+                                        {
+                                            "name": "envValue",
+                                            "type": "env_secret"
                                         }
                                     ],
                                     "outputs": [
@@ -127,7 +155,7 @@
                                             "command": [
                                                 "sh",
                                                 "-c",
-                                                "echo value $0 is huge > /tmp/out"
+                                                "echo env $envValue; echo value $0 is huge > /tmp/out"
                                             ],
                                             "args": []
                                         },
@@ -153,6 +181,10 @@
                                             "name": "valParam",
                                             "mediatype": ["number"],
                                             "type": "parameter"
+                                        },
+                                        {
+                                            "name": "envValue",
+                                            "type": "env_secret"
                                         }
                                     ],
                                     "outputs": [
@@ -167,7 +199,7 @@
                                             "command": [
                                                 "sh",
                                                 "-c",
-                                                "echo value $0 is small > /tmp/out"
+                                                "echo env $envValue; echo value $0 is small > /tmp/out"
                                             ],
                                             "args": []
                                         },

transpiler/helpers.go

@@ -52,8 +52,7 @@ func getNodeSecretMap(targetNodeId string, cmpSecrets secretMap, cmpInputs []mod
 	for _, m := range inputsMap {
 		mt, ok := checkInputType(cmpInputs, m.Source.Port)
 		if ok && mt == models.FlowifySecretType && m.Target.Node == targetNodeId {
-			k, _ := findKeyFor(nSecrets, m.Source.Port)
-			nSecrets[k] = m.Target.Port
+			nSecrets[m.Target.Port] = cmpSecrets[m.Source.Port]
 		}
 	}
 	return nSecrets

transpiler/transpiler.go

@@ -72,9 +72,9 @@ func AddBrick(name string, cmp *models.Brick, outputs wfv1.Outputs, inputs wfv1.
 	container.Args = args
 	envs := []corev1.EnvVar{}
 	for k, e := range sMap {
-		s := corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: secret.DefaultObjectName}, Key: k}
+		s := corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: secret.DefaultObjectName}, Key: e}
 		vf := corev1.EnvVarSource{SecretKeyRef: &s}
-		env := corev1.EnvVar{Name: e, ValueFrom: &vf}
+		env := corev1.EnvVar{Name: k, ValueFrom: &vf}
 		envs = append(envs, env)
 	}
 	container.Env = envs
@@ -155,7 +155,7 @@ func AddGraph(name string, cmp *models.Graph, outputs wfv1.Outputs, inputs wfv1.
 						errors.Errorf("Unrecognized implementation type: %s", impCmp)
 					}
 				case models.FlowifyVolumeType:
-					logrus.Info("Ref volume edge: ", e)
+					logrus.Debug("Ref volume edge:", e)
 				default:
 					return errors.Errorf("Unrecognized edge target-type, '%s', for target port-address '%s.%s'", it, e.Target.Node, e.Target.Port)
 				}
@@ -425,12 +425,11 @@ func TraverseComponent(cmp *models.Component, templates *[]wfv1.Template, tasks
 			}
 			inParams = append(inParams, param)
 		case models.FlowifySecretType:
-			_, ok := findKeyFor(cmpSecrets, i.Name)
-			if !ok {
-				cmpSecrets[i.Name] = i.Name
+			if _, ok := cmpSecrets[i.Name]; !ok {
+				return nil, fmt.Errorf("missing secret <%s> for component: %s", i.Name, cmp.Uid)
 			}
 		case models.FlowifyVolumeType:
-			logrus.Info("Ref mount: ", i.Name, volumes[i.Name])
+			logrus.Debugf("Ref mount: %s %s", i.Name, volumes[i.Name])
 		default:
 			return nil, fmt.Errorf("cannot append input data (name: %s, type %s) at node %s", i.Name, i.Type, cmpName)
 		}
@@ -468,8 +467,7 @@ func TraverseComponent(cmp *models.Component, templates *[]wfv1.Template, tasks
 			if mt, ok := checkInputType(cmp.Inputs, m.Source.Port); ok {
 				switch mt {
 				case models.FlowifySecretType:
-					k, _ := findKeyFor(nodeSecrets, m.Source.Port)
-					nodeSecrets[k] = m.Target.Port
+					nodeSecrets[m.Target.Port] = cmpSecrets[m.Source.Port]
 				case models.FlowifyVolumeType:
 					if val, ok := volumes[m.Source.Port]; ok {
 						nodeVolumes[m.Target.Port] = val
@@ -508,12 +506,11 @@ func TraverseComponent(cmp *models.Component, templates *[]wfv1.Template, tasks
 			if mt, ok := checkInputType(cmp.Inputs, m.Source.Port); ok {
 				switch mt {
 				case models.FlowifySecretType:
-					k, _ := findKeyFor(nodeSecrets, m.Source.Port)
-					nodeSecrets[k] = m.Target.Port
+					nodeSecrets[m.Target.Port] = cmpSecrets[m.Source.Port]
 				case models.FlowifyVolumeType:
 					if val, ok := volumes[m.Source.Port]; ok {
 						nodeVolumes[m.Target.Port] = val
-						logrus.Infof("Rewriting nodeVolumes for %s: %s -> %s", cmp.Name, m.Source.Port, m.Target.Port)
+						logrus.Debugf("Rewriting nodeVolumes for %s: %s -> %s", cmp.Name, m.Source.Port, m.Target.Port)
 					}
 				}
 			}
@@ -538,7 +535,7 @@ func TraverseComponent(cmp *models.Component, templates *[]wfv1.Template, tasks
 							scopeVolumes[m.Target.Node] = make(map[string]corev1.Volume, 0)
 						}
 						scopeVolumes[m.Target.Node][m.Target.Port] = val
-						logrus.Infof("Rewriting nodeVolumes for %s: %s -> %s", cmp.Name, m.Source.Port, m.Target.Port)
+						logrus.Debugf("Rewriting nodeVolumes for %s: %s -> %s", cmp.Name, m.Source.Port, m.Target.Port)
 					}
 				}
 			}
@@ -551,17 +548,17 @@ func TraverseComponent(cmp *models.Component, templates *[]wfv1.Template, tasks
 			}
 			nodeSecrets := getNodeSecretMap(c.Id, cmpSecrets, cmp.Inputs, impCmp.InputMappings)
 			volumes := getConnectedVolumeMap(c.Id, &tc, impCmp.Edges, impCmp.Nodes, scopeVolumes)
-			logrus.Info(volumes)
+			logrus.Debug(volumes)
 			_, err := TraverseComponent(&tc, templates, tasks, nodeSecrets, volumes)
 			if err != nil {
 				return nil, err
 			}
 		}
 	case models.Brick:
-		for k, elem := range cmpSecrets {
+		for k, _ := range cmpSecrets {
 			ex := false
 			for _, i := range cmp.Inputs {
-				if elem == i.Name {
+				if k == i.Name {
 					ex = true
 					break
 				}
@@ -605,15 +602,20 @@ func GetArgoWorkflow(job models.Job) (*wfv1.Workflow, error) {
 	secretMapValues := make(secretMap)
 	for _, cmpInput := range wf.Component.Inputs {
 		if cmpInput.Type == models.FlowifySecretType {
+			missing := true
 			for _, jobInput := range job.InputValues {
 				if cmpInput.Name == jobInput.Target {
 					val, ok := jobInput.Value.(string)
 					if !ok {
 						return nil, errors.Errorf("Cannot convert flowify secret '%s' to string.", jobInput.Target)
 					}
-					secretMapValues[val] = cmpInput.Name
+					secretMapValues[cmpInput.Name] = val
+					missing = false
 				}
 			}
+			if missing {
+				return nil, errors.Errorf("Missing input secret <%s> for job: %s", cmpInput.Name, job.Name)
+			}
 		}
 	}
 
@@ -637,7 +639,7 @@ func GetArgoWorkflow(job models.Job) (*wfv1.Workflow, error) {
 					if err != nil {
 						return nil, err
 					}
-					logrus.Infof("Appending volume from config: %s -> (%s) ", config, v.Target)
+					logrus.Debugf("Appending volume from config: %s -> (%s) ", config, v.Target)
 					volumeMap[tgt.Name] = volume // add volume with top level input name
 				} else {
 					return nil, fmt.Errorf("mount config must be json encoded string")

transpiler/transpiler_test.go

@@ -187,6 +187,8 @@ const (
 	}`
 )
 
+var secrets = secretMap{"secretWF1": "wf1_secret_value", "secretWF2": "wf2_secret_value"}
+
 func init() {
 }
 
@@ -202,7 +204,7 @@ func Test_ParseComponentTree(t *testing.T) {
 	assert.Equal(t, 3, len(wf.Component.Inputs))
 	assert.Equal(t, 0, len(wf.Component.Outputs))
 
-	argoWF, err := ParseComponentTree(wf, secretMap{}, volumeMap{}, map[string]string{}, map[string]string{})
+	argoWF, err := ParseComponentTree(wf, secrets, volumeMap{}, map[string]string{}, map[string]string{})
 	assert.Nil(t, err)
 	assert.True(t, len(argoWF.Spec.Templates) == 4, "Expected no. of templates is 4.")
 	// assert.Equal(t, "seedWF", argoWF.Spec.Arguments.Parameters[0].Name)
@@ -222,17 +224,17 @@ func Test_ParseComponentTree(t *testing.T) {
 			for _, env := range template.Container.Env {
 				switch env.Name {
 				case "secretN1":
-					assert.Equal(t, "secretWF1", env.ValueFrom.SecretKeyRef.Key)
+					assert.Equal(t, secrets["secretWF1"], env.ValueFrom.SecretKeyRef.Key)
 				case "secretN2":
-					assert.Equal(t, "secretWF2", env.ValueFrom.SecretKeyRef.Key)
+					assert.Equal(t, secrets["secretWF2"], env.ValueFrom.SecretKeyRef.Key)
 				default:
 					t.Errorf("Unexpected secret %s in brick %s.", env.Name, template.Name)
 				}
 			}
 		case "192161d7-e3f2-4991-adc0-a99c88c144b2":
 			assert.Equal(t, 1, len(template.Container.Env))
 			assert.Equal(t, "secretN1", template.Container.Env[0].Name)
-			assert.Equal(t, "secretWF2", template.Container.Env[0].ValueFrom.SecretKeyRef.Key)
+			assert.Equal(t, secrets["secretWF2"], template.Container.Env[0].ValueFrom.SecretKeyRef.Key)
 		default:
 			t.Errorf("Unexpected template %s.", template.Name)
 		}
@@ -244,7 +246,8 @@ func Test_RemoveDuplicatedTemplates(t *testing.T) {
 	err := json.Unmarshal([]byte(minimalExampleJSON), &wf)
 	assert.Nil(t, err)
 
-	job := models.Job{Metadata: models.Metadata{Description: "test job"}, Type: "job", InputValues: nil, Workflow: wf}
+	inputValues := []models.Value{{Value: secrets["secretWF1"], Target: "secretWF1"}, {Value: secrets["secretWF2"], Target: "secretWF2"}}
+	job := models.Job{Metadata: models.Metadata{Description: "test job"}, Type: "job", InputValues: inputValues, Workflow: wf}
 	argoWF, err := GetArgoWorkflow(job)
 	assert.Nil(t, err)
 	assert.True(t, len(argoWF.Spec.Templates) == 3, "Expected no. of templates is 3.")